spot_build 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c740d2bdc6f1ab910728822d03e4a857e4ffc144f50c5bcb4d15dd402c730d76
-  data.tar.gz: 167ece42931edd0c434d00eb9409714fe6a0458160da2397f68f89719d2d2bb0
+  metadata.gz: b908bf73e6792eb7a8eec7330ff361f0b169a2f41f73b165e2c6aa7974abc579
+  data.tar.gz: b022e0447aa52a5653b857d18e6b43faa7478d7d862149af1f3a149484e8852a
 SHA512:
-  metadata.gz: d62f1d929dfeef74c78e176d405cae7ba6acb536042b273100edc2a19f79c451ef2fd260f3fb1d8a8ed1797254e87a7e5b57fdccbc12ac3376a964e828f00239
-  data.tar.gz: 7a037186b55880955ff82fa4101e126137b2e8789d19a6d8421d3ce151ea430dc586ec1a3896026f4510339f1dbfee7734fa21e8f360b69260d608c5dc298d70
+  metadata.gz: 7a5c89ebd53903534085a0fb4b0d8530b31460bab4c53aa09ffbae8036ab1eb47e9fa36818747b02974396a776be00dba2db9389b2be8aad919a53d60dbf7ff9
+  data.tar.gz: e4c5fd24ad5d37ec89b5ed4f658f1062e81f02f7e8c5e6ac8f438029596141001eadf1c8e633b6cd43ae61f757489078397d601e35c095f5c4aad363faf2c772

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/lib/spot_build.rb

@@ -1,4 +1,4 @@
-require 'spot_build/buildkite_agent'
+require 'spot_build/buildkite_agents'
 require 'spot_build/spot_instance'
 require 'spot_build/sqs_event'
 require 'optparse'
@@ -15,19 +15,25 @@ module SpotBuild
       checks.push(SqsEvent.new(url: options[:queue_url], timeout: options[:timeout], region: options[:aws_region]))
     end
 
-    agent = BuildkiteAgent.new(options[:token], options[:org_slug])
+    agents = BuildkiteAgents.new(options[:token], options[:org_slug])
     loop do
       checks.each do |check|
         terminating = check.shutdown_if_required do
-          timeout = SpotInstance.scheduled_for_termination? ? (SpotInstance.time_until_termination - 30) : options[:timeout]
-
-          agent.stop
-          Timeout::timeout(timeout) do
-            while agent.agent_running?
+          agents.stop
+          if options[:auto_retries]
+            timeout = SpotInstance.scheduled_for_termination? ? (SpotInstance.time_until_termination - 30) : options[:timeout]
+
+            Timeout::timeout(timeout) do
+              while agents.agents_running?
+                sleep 5
+              end
+            end rescue Timeout::Error
+            agents.the_end_is_nigh
+          else
+            while agents.agents_running?
               sleep 5
             end
-          end rescue Timeout::Error
-          agent.the_end_is_nigh
+          end
         end
         %x(shutdown -h now) if terminating
       end
@@ -36,7 +42,7 @@ module SpotBuild
   end
 
   def self.parse_options
-    options = {}
+    options = {auto_retries: true}
     parser = OptionParser.new do |opts|
       opts.banner = "Usage: #{__FILE__} [options]"
       opts.on("-t", "--token TOKEN", "Buildkite API token") { |v| options[:token] = v }
@@ -44,6 +50,7 @@ module SpotBuild
       opts.on("-s", "--sqs-queue SQS-QUEUE-URL", "The SQS Queue URL we should monitor for events that tell us to shutdown") { |v| options[:queue_url] = v }
       opts.on("--timeout TIMEOUT", "The amount of time to wait for the buildkite agent to stop before shutting down. Only used if --sqs-queue is specified") { |v| options[:timeout] = v.to_i }
       opts.on("-r", "--aws-region REGION", "The AWS Region the SQS queue resides in")  { |v| options[:aws_region] = v }
+      opts.on("-n", "--[no-]auto-retry", "Disable automatic retries") { |v| options[:auto_retries] = v }
     end
     parser.parse!
 

data/lib/spot_build/{buildkite_agent.rb → buildkite_agents.rb}

@@ -3,35 +3,62 @@ require 'socket'
 require 'link_header'
 
 module SpotBuild
-  class BuildkiteAgent
+  class BuildkiteAgents
     def initialize(token, org_slug)
       @client = Buildkit.new(token: token)
       @org_slug = org_slug
     end
 
-    def the_end_is_nigh
-      return unless agent_running?
-      job = current_job
-      stop(true)
-      reschedule_job(job)
+    def the_end_is_nigh(host = Socket.gethostname)
+      agents = agents_on_this_host(host)
+      agents.each do |agent|
+        stop_agent(agent, force: true)
+      end
+      agents.each do |agent|
+        reschedule_job(agent.job)
+      end
+      agents.count
     end
 
-    def stop(force="false")
-      return unless agent_running?
-      @client.stop_agent(@org_slug, agent_id, "{\"force\": #{force}}")
+    def stop_agent(agent, force: false)
+      @client.stop_agent(@org_slug, agent.id, "{\"force\": #{force}}")
     rescue Buildkit::UnprocessableEntity
       # Swallow the error, this is generally thrown when the agent has already stopped
     end
 
-    def agent_running?
-      !agent.nil?
+    def agents_running?(host = Socket.gethostname)
+      !agents_on_this_host(host).empty?
+    end
+
+    def stop(host = Socket.gethostname)
+      agents_on_this_host(host).each do |agent|
+        stop_agent(agent, force: false)
+      end
     end
 
     private
 
+    RETRY_MESSAGE = /Only failed or timed out jobs can be retried/.freeze
+
     def reschedule_job(job)
       return if job.nil?
-      @client.retry_job(@org_slug, job_pipeline(job[:build_url]), job_build(job[:build_url]), job[:id])
+      retry_error(Buildkit::BadRequest, RETRY_MESSAGE) do
+        @client.retry_job(@org_slug, job_pipeline(job[:build_url]), job_build(job[:build_url]), job[:id])
+      end
+    end
+
+    def retry_error(error_class, message_regex, sleep_seconds: 1, retries: 20)
+      begin
+        yield
+      rescue error_class => e
+        if retries > 0 && e.message =~ message_regex
+          sleep sleep_seconds
+          retries -= 1
+          retry
+        else
+          raise
+        end
+      end
     end
 
     # build_url: https://api.buildkite.com/v2/organizations/my-great-org/pipelines/sleeper/builds/50
@@ -43,19 +70,11 @@ module SpotBuild
       build_url[%r{organizations/#{@org_slug}/pipelines/[^/]*/builds/([0-9]*)}, 1]
     end
 
-    def current_job
-      agent.job
-    end
-
-    def agent_id
-      @agent_id ||= agent.id
-    end
-
-    def agent
-      agents.select { |agent| agent.hostname == Socket.gethostname }.first
+    def agents_on_this_host(host)
+      all_agents.select { |agent| agent.hostname == host }
     end
 
-    def agents
+    def all_agents
       with_pagination do |options = {}|
         @client.agents(@org_slug, options)
       end

data/spec/buildkite_agent_spec.rb

@@ -1,12 +1,21 @@
 require 'spec_helper'
 
-describe SpotBuild::BuildkiteAgent do
+describe SpotBuild::BuildkiteAgents do
   let(:org_slug) { "envato" }
-  subject { described_class.new('deadbeef', org_slug) }
-  
+  let(:pipeline) { "my-app" }
+  subject(:buildkite_agent) { described_class.new('deadbeef', org_slug) }
   let(:last_response_stub) { instance_double(Sawyer::Response) }
-  let(:buildkit_stub) { double("Buildkit", :agents => agent_stubs) }
+  let(:buildkit_stub) { instance_double("Buildkit::Client", :agents => agent_stubs) }
   let(:hostname) { "i-1234567890" }
+  let(:build_id) { "12345678" }
+
+  def agent(id:, build_id: "12345678", job_id: "1")
+    double("BuildkiteAgent#{id}",
+      hostname: hostname,
+      id: id,
+      job: {build_url: "organizations/#{org_slug}/pipelines/#{pipeline}/builds/#{build_id}", id: job_id}
+    )
+  end
 
   before do
     allow(Buildkit).to receive(:new).and_return(buildkit_stub)
@@ -15,50 +24,80 @@ describe SpotBuild::BuildkiteAgent do
     allow(last_response_stub).to receive(:headers).and_return({"link" => nil})
   end
 
+  describe '#agents_running?' do
+    context 'when agents are running' do
+      let(:agent_stubs) { [agent(id: '123', build_id: build_id, job_id: '1')] }
+
+      it 'returns true' do
+        expect(buildkite_agent.agents_running?).to eq true
+      end
+    end
+
+    context "when agents aren't running" do
+      let(:agent_stubs) { [] }
+
+      it 'returns false' do
+        expect(buildkite_agent.agents_running?).to eq false
+      end
+    end
+  end
+
   describe '#the_end_is_nigh' do
     context 'the agent is not running' do
       let(:agent_stubs) { [] }
 
-      it 'returns nil' do
-        expect(subject.the_end_is_nigh).to equal(nil)
+      it 'does nothing' do
+        expect(buildkit_stub).to_not receive(:stop_agent)
+        expect(buildkit_stub).to_not receive(:retry_job)
+        buildkite_agent.the_end_is_nigh
       end
     end
 
-    context 'the agent is running' do
-      let(:agent_id) { 9876 }
-      let(:agent_stubs) { 
-        [double("BuildkiteAgent",
-          hostname: hostname,
-          id: agent_id,
-          job: {build_url: "organizations/#{org_slug}/pipelines/my-app/builds/12345678", id: "12345678"}
-        )]
-      }
+    context 'agents are running' do
+      let(:agent_1_id) { '9876' }
+      let(:agent_2_id) { '9877' }
+      let(:agent_stubs) { [agent(id: agent_1_id, build_id: build_id, job_id: '1'),
+                           agent(id: agent_2_id, build_id: build_id, job_id: '2')] }
 
       before do
         allow(buildkit_stub).to receive(:stop_agent)
         allow(buildkit_stub).to receive(:retry_job)
       end
 
-      it 'stops the agent forcefully' do
-        expect(buildkit_stub).to receive(:stop_agent).with(org_slug, agent_id, '{"force": true}')
-        subject.the_end_is_nigh
+      it 'stops each agent forcefully' do
+        expect(buildkit_stub).to receive(:stop_agent).with(org_slug, agent_1_id, '{"force": true}')
+        expect(buildkit_stub).to receive(:stop_agent).with(org_slug, agent_2_id, '{"force": true}')
+        buildkite_agent.the_end_is_nigh
       end
 
       it 'reschedules the job' do
-        expect(buildkit_stub).to receive(:retry_job)
-        subject.the_end_is_nigh
+        expect(buildkit_stub).to receive(:retry_job).with(org_slug, pipeline, build_id, '1')
+        expect(buildkit_stub).to receive(:retry_job).with(org_slug, pipeline, build_id, '2')
+        buildkite_agent.the_end_is_nigh
+      end
+
+      context "when the jobs aren't retryable yet" do
+        let(:agent_stubs) { [agent(id: agent_1_id, build_id: build_id, job_id: '1')] }
+
+        it 'retries' do
+          responses = [
+            -> { raise Buildkit::BadRequest, {method: 'PUT', url: 'https://api.buildkite.com/v2/organizations/#{org_slug}/pipelines/#{pipeline}/builds/18961/jobs/1/retry', body: 'Only failed or timed out jobs can be retried'} },
+            -> { nil }
+          ]
+          allow(buildkit_stub).to receive(:retry_job).with(org_slug, pipeline, build_id, '1') do
+            response = responses.shift
+            response.call if response
+          end
+          buildkite_agent.the_end_is_nigh
+          expect(buildkit_stub).to have_received(:retry_job)
+            .with(org_slug, pipeline, build_id, '1')
+            .twice
+        end
       end
     end
 
     context 'the agent stops while we are trying to stop it' do
-      let(:agent_id) { 9876 }
-      let(:agent_stubs) { 
-        [double("BuildkiteAgent",
-          hostname: hostname,
-          id: agent_id,
-          job: {build_url: "organizations/#{org_slug}/pipelines/my-app/builds/12345678", id: "12345678"}
-        )]
-      }
+      let(:agent_stubs) { [agent(id: '9876')] }
 
       before do
         allow(buildkit_stub).to receive(:stop_agent).and_raise(Buildkit::UnprocessableEntity)
@@ -67,7 +106,7 @@ describe SpotBuild::BuildkiteAgent do
 
       it 'retries the job' do
         expect(buildkit_stub).to receive(:retry_job)
-        subject.the_end_is_nigh
+        buildkite_agent.the_end_is_nigh
       end
     end
   end

data/spot_build.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = 'spot_build'
-  gem.version       = '1.0.0'
+  gem.version       = '1.1.0'
   gem.authors       = ['Patrick Robinson']
   gem.email         = []
   gem.description   = 'Helps manage Buildkite Agents running on EC2 Spot instances'
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
-  gem.add_dependency 'buildkit', '~> 0.4'
+  gem.add_dependency 'buildkit', '~> 1.4'
   gem.add_dependency 'aws-sdk', '~> 2'
   gem.add_dependency 'link_header', '~> 0.0.2'
   gem.add_development_dependency 'rspec',  '~> 3'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spot_build
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Patrick Robinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-19 00:00:00.000000000 Z
+date: 2019-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: buildkit
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -87,16 +87,16 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - ".ruby-version"
 - ".travis.yml"
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/spot_build
 - lib/spot_build.rb
-- lib/spot_build/buildkite_agent.rb
+- lib/spot_build/buildkite_agents.rb
 - lib/spot_build/spot_instance.rb
 - lib/spot_build/sqs_event.rb
 - spec/buildkite_agent_spec.rb
@@ -120,8 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Helps manage Buildkite Agents running on EC2 Spot instances

data/Gemfile.lock

@@ -1,56 +0,0 @@
-PATH
-  remote: .
-  specs:
-    spot_build (0.0.4)
-      aws-sdk (~> 2)
-      buildkit (~> 0.4)
-      link_header (~> 0.0.2)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.3.8)
-    aws-sdk (2.11.33)
-      aws-sdk-resources (= 2.11.33)
-    aws-sdk-core (2.11.33)
-      aws-sigv4 (~> 1.0)
-      jmespath (~> 1.0)
-    aws-sdk-resources (2.11.33)
-      aws-sdk-core (= 2.11.33)
-    aws-sigv4 (1.0.2)
-    buildkit (0.4.0)
-      sawyer (~> 0.6.0)
-    diff-lcs (1.3)
-    faraday (0.9.2)
-      multipart-post (>= 1.2, < 3)
-    jmespath (1.4.0)
-    link_header (0.0.8)
-    multipart-post (2.0.0)
-    rake (12.3.1)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.1)
-    sawyer (0.6.0)
-      addressable (~> 2.3.5)
-      faraday (~> 0.8, < 0.10)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  rake
-  rspec (~> 3)
-  spot_build!
-
-BUNDLED WITH
-   1.16.1