spot_build 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 58c259e6f3b6c9e4ccbc8682426d3ab23b257406
+  data.tar.gz: 9a6d4d033c24ab2778946281c967e359968e9f12
+SHA512:
+  metadata.gz: bd29f93748e8113a7c31ad592eeef931cecdd6b75dc40884760013261e62d0138fd3dad58ec556f2852fbf4900173c432acb32f2b0405566f2bdee1db582d758
+  data.tar.gz: 9db1142b19e9fcd33367394df0bccafad2bb655e8ba37ff9d7ac46864687726bfcdb60f9ccf53c2c51352cd8d3129cf5d20b93cccd2271bb297bdeb9e93fec05

data/.ruby-version

@@ -0,0 +1 @@
+2.3.0

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,39 @@
+PATH
+  remote: .
+  specs:
+    spot_build (0.0.2)
+      aws-sdk (~> 2)
+      buildkit (~> 0.4)
+      link_header (~> 0.0.2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    aws-sdk (2.6.38)
+      aws-sdk-resources (= 2.6.38)
+    aws-sdk-core (2.6.38)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.6.38)
+      aws-sdk-core (= 2.6.38)
+    aws-sigv4 (1.0.0)
+    buildkit (0.4.0)
+      sawyer (~> 0.6.0)
+    faraday (0.9.2)
+      multipart-post (>= 1.2, < 3)
+    jmespath (1.3.1)
+    link_header (0.0.8)
+    multipart-post (2.0.0)
+    sawyer (0.6.0)
+      addressable (~> 2.3.5)
+      faraday (~> 0.8, < 0.10)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  spot_build!
+
+BUNDLED WITH
+   1.13.6

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Envato
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+# Managing Spot Fleets with Buildkite
+
+AWS EC2 Spot instances are cheaper, Buildkite Agents are a natural fit for Spot instnaces as the workload is interruptable.
+
+SpotBuild makes it easier to use Spot instances and Spot fleets with Buildkite Agents by providing an agent that will shutdown the agent when the instance is scheduled for termination, preventing it from starting any new jobs and retry the current job it's working on.
+
+# Running
+
+Run this gem as a daemon on your buildkite agents and supply it the Organisation Slug and a Buildkite API token with the following permissions:
+- read_agents
+- read_builds
+- write_builds
+
+## Development Status
+
+Very early stages of development

data/bin/spot_build

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'spot_build'
+
+SpotBuild.run

data/lib/spot_build/buildkite_agent.rb

@@ -0,0 +1,77 @@
+require 'buildkit'
+require 'socket'
+require 'link_header'
+
+module SpotBuild
+  class BuildkiteAgent
+    def initialize(token, org_slug)
+      @client = Buildkit.new(token: token)
+      @org_slug = org_slug
+    end
+
+    def the_end_is_nigh
+      return unless agent_running?
+      job = current_job
+      stop(true)
+      reschedule_job(job)
+    end
+
+    def stop(force="false")
+      return unless agent_running?
+      @client.stop_agent(@org_slug, agent_id, "{\"force\": #{force}}")
+    end
+
+    def agent_running?
+      !agent.nil?
+    end
+
+    private
+
+    def reschedule_job(job)
+      return if job.nil?
+      @client.retry_job(@org_slug, job_pipeline(job[:build_url]), job_build(job[:build_url]), job[:id])
+    end
+
+    # build_url: https://api.buildkite.com/v2/organizations/my-great-org/pipelines/sleeper/builds/50
+    def job_pipeline(build_url)
+      build_url[%r{organizations/#{@org_slug}/pipelines/([^/]*)}, 1]
+    end
+
+    def job_build(build_url)
+      build_url[%r{organizations/#{@org_slug}/pipelines/[^/]*/builds/([0-9]*)}, 1]
+    end
+
+    def current_job
+      agent[:job]
+    end
+
+    def agent_id
+      @agent_id ||= agent[:id]
+    end
+
+    def agent
+      agents.select { |agent| agent.hostname == Socket.gethostname }.first
+    end
+
+    def agents
+      with_pagination do |options = {}|
+        @client.agents(@org_slug, options)
+      end
+    end
+
+    # This is definately not thread safe
+    def with_pagination(&block)
+      results = yield
+      while next_ref = next_link_ref(@client.last_response.headers["link"])
+        uri = URI.parse(next_ref.href)
+        next_page = uri.query.split("=")[1]
+        results.push(yield page: next_page)
+      end
+      results.flatten
+    end
+
+    def next_link_ref(header)
+      LinkHeader.parse(header).find_link(["rel", "next"])
+    end
+  end
+end

data/lib/spot_build/spot_instance.rb

@@ -0,0 +1,25 @@
+require 'net/http'
+require 'time'
+
+module SpotBuild
+  class SpotInstance
+    def shutdown_if_required(&block)
+      return false unless self.class.scheduled_for_termination?
+      yield
+      true
+    end
+
+    def self.scheduled_for_termination?
+      !time_until_termination.nil?
+    end
+
+    def self.time_until_termination
+      uri = URI('http://169.254.169.254/latest/meta-data/spot/termination-time')
+      response = Net::HTTP.get_response(uri)
+      return nil if response.code == "404"
+      Time.parse(response.body) - Time.now
+    rescue ArgumentError
+      nil
+    end
+  end
+end

data/lib/spot_build/sqs_event.rb

@@ -0,0 +1,24 @@
+require 'aws-sdk'
+require 'timeout'
+
+module SpotBuild
+  class SqsEvent
+    def initialize(url:, timeout:, region: ENV['AWS_REGION'])
+      @queue = Aws::SQS::Queue.new(url: url, region: region)
+      @timeout = timeout
+    end
+
+    def shutdown_if_required(&block)
+      # Any message to this queue is treated as a "I should shutdown"
+      message = @queue.receive_messages(
+        attribute_names: ["All"],
+        max_number_of_messages: 1,
+        visibility_timeout: (@timeout - 5),
+      ).first
+      return false if message.nil?
+      yield
+      message.delete
+      true
+    end
+  end
+end

data/lib/spot_build.rb

@@ -0,0 +1,56 @@
+require 'spot_build/buildkite_agent'
+require 'spot_build/spot_instance'
+require 'spot_build/sqs_event'
+require 'optparse'
+
+module SpotBuild
+  DEFAULT_TIMEOUT = 300
+
+  def self.run
+    options = parse_options
+    options[:timeout] ||= DEFAULT_TIMEOUT
+
+    checks = [SpotInstance.new]
+    if options[:queue_url]
+      checks.push(SqsEvent.new(url: options[:queue_url], timeout: options[:timeout], region: options[:aws_region]))
+    end
+
+    agent = BuildkiteAgent.new(options[:token], options[:org_slug])
+    loop do
+      checks.each do |check|
+        terminating = check.shutdown_if_required do
+          timeout = SpotInstance.scheduled_for_termination? ? (SpotInstance.time_until_termination - 30) : options[:timeout]
+
+          agent.stop
+          Timeout::timeout(timeout) do
+            while agent.agent_running?
+              sleep 5
+            end
+          end rescue Timeout::Error
+          agent.the_end_is_nigh
+        end
+        %x(shutdown -h now) if terminating
+      end
+      sleep 2
+    end
+  end
+
+  def self.parse_options
+    options = {}
+    parser = OptionParser.new do |opts|
+      opts.banner = "Usage: #{__FILE__} [options]"
+      opts.on("-t", "--token TOKEN", "Buildkite API token") { |v| options[:token] = v }
+      opts.on("-o", "--org-slug ORGANISATION-SLUG", "The Buildkite Organisation Slug") { |v| options[:org_slug] = v }
+      opts.on("-s", "--sqs-queue SQS-QUEUE-URL", "The SQS Queue URL we should monitor for events that tell us to shutdown") { |v| options[:queue_url] = v }
+      opts.on("--timeout TIMEOUT", "The amount of time to wait for the buildkite agent to stop before shutting down. Only used if --sqs-queue is specified") { |v| options[:timeout] = v.to_i }
+      opts.on("-r", "--aws-region REGION", "The AWS Region the SQS queue resides in")  { |v| options[:aws_region] = v }
+    end
+    parser.parse!
+
+    if options[:token].nil? || options[:org_slug].nil?
+      raise OptionParser::MissingArgument, "You must specify Token and Organisational Slug.\n#{parser.help}"
+    end
+
+    options
+  end
+end

data/spot_build.gemspec

@@ -0,0 +1,21 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = 'spot_build'
+  gem.version       = '0.0.4'
+  gem.authors       = ['Patrick Robinson']
+  gem.email         = []
+  gem.description   = 'Helps manage Buildkite Agents running on EC2 Spot instances'
+  gem.summary       = gem.description
+  gem.homepage      = 'https://github.com/envato/spot_build'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'buildkit', '~> 0.4'
+  gem.add_dependency 'aws-sdk', '~> 2'
+  gem.add_dependency 'link_header', '~> 0.0.2'
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: spot_build
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Patrick Robinson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: buildkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: link_header
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+description: Helps manage Buildkite Agents running on EC2 Spot instances
+email: []
+executables:
+- spot_build
+extensions: []
+extra_rdoc_files: []
+files:
+- .ruby-version
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- bin/spot_build
+- lib/spot_build.rb
+- lib/spot_build/buildkite_agent.rb
+- lib/spot_build/spot_instance.rb
+- lib/spot_build/sqs_event.rb
+- spot_build.gemspec
+homepage: https://github.com/envato/spot_build
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14.1
+signing_key: 
+specification_version: 4
+summary: Helps manage Buildkite Agents running on EC2 Spot instances
+test_files: []