nomade 0.0.5 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85557a8305dae1d56eaf0f9d8497948f928f270b8f43d28860836a1ad1fb0b3e
-  data.tar.gz: 92e9afd8c92efb363edf7daf7c3f04f40dd71ca243c58dddfd9d4cd71925bd65
+  metadata.gz: c6c3684724566430b8c0af280a5be1e03be57a5c7bcac2298c22d14d03098565
+  data.tar.gz: ae59d84da529ab3e4041d52a9216f0cf4e68cd7cd39751d895907d05f52b19cf
 SHA512:
-  metadata.gz: 0f33f910a59f67c4dcf63ba394a4d4a6d3e48ea555a562d73f5ee2d0dc59e98489270b9502a5e3dc8a2f56398a4d605e0ee11a135d26332a74a5186e3b162d44
-  data.tar.gz: 172af74987917d6a0b2cea45d2e461991a5bc7af46de64a738edd220ecd0e829a25293013fcc45e864c1e7e4eeb74cbcc4da813b86188403a57016bb3f813266
+  metadata.gz: 1fb1d40654d23c00694ca29e9423050e3e829dd543dd293b5655c7def2885fa0c73593bc06a35a7f3c7f53869b3f4afc470077d9e9613cb8b0d113d1328ee49f
+  data.tar.gz: be3a100801b40b10a38f162095a86f3f36e386b9ff17e46ee7597e7ab37c373e059c8472acbfa6262bd56d37743252d5bb7780925812cfef6227a7f18c63d54f

data/lib/nomade/deployer.rb

@@ -1,66 +1,103 @@
 module Nomade
   class Deployer
-    def initialize(nomad_endpoint, nomad_job, opts = {})
-      @nomad_job = nomad_job
-      @evaluation_id = nil
-      @deployment_id = nil
-      @timeout = Time.now.utc + 60 * 3 # minutes
+    attr_reader :nomad_job
+
+    def initialize(nomad_endpoint, opts = {})
       @nomad_endpoint = nomad_endpoint
       @http = Nomade::Http.new(@nomad_endpoint)
+      @job_builder = Nomade::JobBuilder.new(@http)
       @logger = opts.fetch(:logger, Nomade.logger)
 
-      @on_success = opts.fetch(:on_success, [])
-      @on_failure = opts.fetch(:on_failure, [])
-      @on_failure << method(:print_errors)
+      @timeout = 60 * 3 # seconds
+
+      @hooks = {
+        Nomade::Hooks::DEPLOY_RUNNING => [],
+        Nomade::Hooks::DEPLOY_FINISHED => [],
+        Nomade::Hooks::DEPLOY_FAILED => [],
+      }
+
+      self
+    end
+
+    def init_job(template_file, image_full_name, template_variables = {})
+      @nomad_job = @job_builder.build(template_file, image_full_name, template_variables)
+      @evaluation_id = nil
+      @deployment_id = nil
+
+      self
+    end
+
+    def add_hook(hook, hook_method)
+      if Nomade::Hooks::DEPLOY_RUNNING == hook
+        @hooks[Nomade::Hooks::DEPLOY_RUNNING] << hook_method
+      elsif Nomade::Hooks::DEPLOY_FINISHED == hook
+        @hooks[Nomade::Hooks::DEPLOY_FINISHED] << hook_method
+      elsif Nomade::Hooks::DEPLOY_FAILED == hook
+        @hooks[Nomade::Hooks::DEPLOY_FAILED] << hook_method
+      else
+        raise "#{hook} not supported!"
+      end
     end
 
     def deploy!
-      plan
-      deploy
+      run_hooks(Nomade::Hooks::DEPLOY_RUNNING, @nomad_job, nil)
+      _plan
+      _deploy
+      run_hooks(Nomade::Hooks::DEPLOY_FINISHED, @nomad_job, nil)
     rescue Nomade::NoModificationsError => e
-      call_failure_handlers ["No modifications to make, exiting!"]
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "No modifications to make, exiting!"])
       exit(0)
     rescue Nomade::GeneralError => e
-      call_failure_handlers [e.message, "GeneralError hit, exiting!"]
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "GeneralError hit, exiting!"])
       exit(1)
-    rescue Nomade::PlanningError => e
-      call_failure_handlers ["Couldn't make a plan, maybe a bad connection to Nomad server, exiting!"]
-      exit(2)
     rescue Nomade::AllocationFailedError => e
-      call_failure_handlers ["Allocation failed with errors, exiting!"]
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "Allocation failed with errors, exiting!"])
       exit(3)
     rescue Nomade::UnsupportedDeploymentMode => e
-      call_failure_handlers [e.message, "Deployment failed with errors, exiting!"]
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "Deployment failed with errors, exiting!"])
       exit(4)
     rescue Nomade::FailedTaskGroupPlan => e
-      call_failure_handlers [e.message, "Couldn't plan correctly, exiting!"]
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "Couldn't plan correctly, exiting!"])
       exit(5)
+    rescue Nomade::DeploymentFailedError => e
+      run_hooks(Nomade::Hooks::DEPLOY_FAILED, @nomad_job, [e.message, "Couldn't deploy succesfully, exiting!"])
+      exit(6)
     end
 
-    private
-
-    def call_failure_handlers(messages)
-      @on_failure.each do |failure_handler|
-        failure_handler.call(messages)
-      end
+    def stop!(purge = false)
+      @http.stop_job(@nomad_job, purge)
     end
 
-    def print_errors(errors)
-      errors.each do |error|
-        @logger.warn(error)
+    private
+
+    def run_hooks(hook, job, messages)
+      @hooks[hook].each do |hook_method|
+        hook_method.call(hook, job, messages)
       end
     end
 
-    def plan
+    def _plan
       @http.capacity_plan_job(@nomad_job)
     end
 
-    def deploy
+    def _deploy
       @logger.info "Deploying #{@nomad_job.job_name} (#{@nomad_job.job_type}) with #{@nomad_job.image_name_and_version}"
       @logger.info "URL: #{@nomad_endpoint}/ui/jobs/#{@nomad_job.job_name}"
 
       @logger.info "Checking cluster for connectivity and capacity.."
-      @http.plan_job(@nomad_job)
+      plan_data = @http.plan_job(@nomad_job)
+
+      sum_of_changes = plan_data["Annotations"]["DesiredTGUpdates"].map { |group_name, task_group_updates|
+        task_group_updates["Stop"] +
+        task_group_updates["Place"] +
+        task_group_updates["Migrate"] +
+        task_group_updates["DestructiveUpdate"] +
+        task_group_updates["Canary"]
+      }.sum
+
+      if sum_of_changes == 0
+        raise Nomade::NoModificationsError.new
+      end
 
       @evaluation_id = if @http.check_if_job_exists?(@nomad_job)
         @logger.info "Updating existing job"
@@ -81,7 +118,14 @@ module Nomade
         sleep(1)
       end
 
-      @logger.info "Waiting until allocations are complete"
+      @logger.info "Waiting until allocations are no longer pending"
+      allocations = @http.allocations_from_evaluation_request(@evaluation_id)
+      until allocations.all?{|a| a["ClientStatus"] != "pending"}
+        @logger.info "."
+        sleep(2)
+        allocations = @http.allocations_from_evaluation_request(@evaluation_id)
+      end
+
       case @nomad_job.job_type
       when "service"
         service_deploy
@@ -106,7 +150,7 @@ module Nomade
           if stdout != ""
             @logger.info
             @logger.info "stdout:"
-            stdout.lines do |logline|
+            stdout.lines.each do |logline|
               @logger.info(logline.strip)
             end
           end
@@ -115,7 +159,7 @@ module Nomade
           if stderr != ""
             @logger.info
             @logger.info "stderr:"
-            stderr.lines do |logline|
+            stderr.lines.each do |logline|
               @logger.info(logline.strip)
             end
           end
@@ -140,7 +184,8 @@ module Nomade
 
     def service_deploy
       @logger.info "Waiting until tasks are placed"
-      @logger.info ".. deploy timeout is #{@timeout}"
+      deploy_timeout = Time.now.utc + @timeout
+      @logger.info ".. deploy timeout is #{deploy_timeout}"
 
       json = @http.deployment_request(@deployment_id)
       @logger.info "#{json["JobID"]} version #{json["JobVersion"]}"
@@ -192,7 +237,7 @@ module Nomade
             succesful_deployment = false
           end
 
-          if succesful_deployment == nil && Time.now.utc > @timeout
+          if succesful_deployment == nil && Time.now.utc > deploy_timeout
             @logger.info "Timeout hit, rolling back deploy!"
             @http.fail_deployment(@deployment_id)
             succesful_deployment = false
@@ -235,9 +280,13 @@ module Nomade
       if succesful_deployment
         @logger.info ""
         @logger.info "#{@deployment_id} (version #{json["JobVersion"]}) was succesfully deployed!"
+
+        true
       else
         @logger.warn ""
         @logger.warn "#{@deployment_id} (version #{json["JobVersion"]}) deployment _failed_!"
+
+        raise DeploymentFailedError.new
       end
     end
 
@@ -286,7 +335,7 @@ module Nomade
               if stdout != ""
                 @logger.info
                 @logger.info "stdout:"
-                stdout.lines do |logline|
+                stdout.lines.each do |logline|
                   @logger.info(logline.strip)
                 end
               end
@@ -295,7 +344,7 @@ module Nomade
               if stderr != ""
                 @logger.info
                 @logger.info "stderr:"
-                stderr.lines do |logline|
+                stderr.lines.each do |logline|
                   @logger.info(logline.strip)
                 end
               end
@@ -307,6 +356,8 @@ module Nomade
 
         sleep(1)
       end
+
+      true
     end
 
     # Task-helpers

data/lib/nomade/exceptions.rb

@@ -1,7 +1,8 @@
 module Nomade
+  class FormattingError < StandardError; end
+
   class GeneralError < StandardError; end
   class NoModificationsError < StandardError; end
-  class PlanningError < StandardError; end
 
   class AllocationFailedError < StandardError
     def initialize(evaluation_id, allocations)
@@ -10,6 +11,9 @@ module Nomade
     end
     attr_reader :evaluation_id, :allocations
   end
+
+  class DeploymentFailedError < StandardError;end
+
   class UnsupportedDeploymentMode < StandardError; end
   class FailedTaskGroupPlan < StandardError; end
 end

data/lib/nomade/hooks.rb

@@ -0,0 +1,7 @@
+module Nomade
+  module Hooks
+    DEPLOY_RUNNING = Class.new
+    DEPLOY_FINISHED = Class.new
+    DEPLOY_FAILED = Class.new
+  end
+end

data/lib/nomade/http.rb

@@ -120,9 +120,10 @@ module Nomade
 
       req = Net::HTTP::Post.new(uri)
       req.add_field "Content-Type", "application/json"
-      req.body = nomad_job.configuration(:json)
+      req.body = JSON.generate({"Job" => nomad_job.configuration(:hash)})
 
       res = http.request(req)
+
       raise if res.code != "200"
       raise if res.content_type != "application/json"
 
@@ -143,7 +144,34 @@ module Nomade
 
       req = Net::HTTP::Post.new(uri)
       req.add_field "Content-Type", "application/json"
-      req.body = nomad_job.configuration(:json)
+      req.body = JSON.generate({"Job" => nomad_job.configuration(:hash)})
+
+      res = http.request(req)
+
+      raise if res.code != "200"
+      raise if res.content_type != "application/json"
+
+      return JSON.parse(res.body)["EvalID"]
+    rescue StandardError => e
+      Nomade.logger.fatal "HTTP Request failed (#{e.message})"
+      raise
+    end
+
+    def stop_job(nomad_job, purge = false)
+      uri = if purge
+        URI("#{@nomad_endpoint}/v1/job/#{nomad_job.job_name}?purge=true")
+      else
+        URI("#{@nomad_endpoint}/v1/job/#{nomad_job.job_name}")
+      end
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      if @nomad_endpoint.include?("https://")
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      end
+
+      req = Net::HTTP::Delete.new(uri)
+      req.add_field "Content-Type", "application/json"
 
       res = http.request(req)
       raise if res.code != "200"
@@ -223,7 +251,22 @@ module Nomade
     end
 
     def capacity_plan_job(nomad_job)
-      uri = URI("#{@nomad_endpoint}/v1/job/#{nomad_job.job_name}/plan")
+      plan_output = plan_job(nomad_job)
+
+      if plan_output["FailedTGAllocs"]
+        raise Nomade::FailedTaskGroupPlan.new("Failed to plan groups: #{plan_output["FailedTGAllocs"].keys.join(",")}")
+      end
+
+      true
+    rescue Nomade::FailedTaskGroupPlan => e
+      raise
+    rescue StandardError => e
+      Nomade.logger.fatal "HTTP Request failed (#{e.message})"
+      raise
+    end
+
+    def convert_hcl_to_json(job_hcl)
+      uri = URI("#{@nomad_endpoint}/v1/jobs/parse")
 
       http = Net::HTTP.new(uri.host, uri.port)
       if @nomad_endpoint.include?("https://")
@@ -233,46 +276,45 @@ module Nomade
 
       req = Net::HTTP::Post.new(uri)
       req.add_field "Content-Type", "application/json"
-      req.body = nomad_job.configuration(:json)
+
+      req.body = JSON.generate({
+        "JobHCL": job_hcl,
+        "Canonicalize": false,
+      })
 
       res = http.request(req)
       raise if res.code != "200"
       raise if res.content_type != "application/json"
 
-      plan_output = JSON.parse(res.body)
-
-      if plan_output["FailedTGAllocs"]
-        raise Nomade::FailedTaskGroupPlan.new("Failed to plan groups: #{plan_output["FailedTGAllocs"].keys.join(",")}")
-      end
-
-      true
-    rescue Nomade::FailedTaskGroupPlan => e
-      raise
+      res.body
     rescue StandardError => e
       Nomade.logger.fatal "HTTP Request failed (#{e.message})"
       raise
     end
 
     def plan_job(nomad_job)
-      rendered_template = nomad_job.configuration(:hcl)
-
-      #   0: No allocations created or destroyed. Nothing to do.
-      #   1: Allocations created or destroyed.
-      # 255: Error determining plan results. Nothing to do.
-      allowed_exit_codes = [0, 1, 255]
-
-      exit_status, stdout, stderr = Shell.exec("NOMAD_ADDR=#{@nomad_endpoint} nomad job plan -diff -verbose -no-color -", rendered_template, allowed_exit_codes)
-
-      case exit_status
-      when 0
-        raise Nomade::NoModificationsError.new
-      when 1
-        # no-op
-      when 255
-        raise Nomade::PlanningError.new
+      uri = URI("#{@nomad_endpoint}/v1/job/#{nomad_job.job_name}/plan")
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      if @nomad_endpoint.include?("https://")
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
       end
 
-      true
+      req = Net::HTTP::Post.new(uri)
+      req.add_field "Content-Type", "application/json"
+      req.body = JSON.generate({"Job" => nomad_job.configuration(:hash)})
+
+      res = http.request(req)
+
+      raise if res.code != "200"
+      raise if res.content_type != "application/json"
+
+      JSON.parse(res.body)
+    rescue StandardError => e
+      Nomade.logger.fatal "HTTP Request failed (#{e.message})"
+      raise
     end
+
   end
 end

data/lib/nomade/job.rb

@@ -1,25 +1,10 @@
-require "erb"
-require "json"
-
 module Nomade
   class Job
-    class FormattingError < StandardError; end
-
-    def initialize(template_file, image_full_name, template_variables = {})
+    def initialize(image_full_name, config_hcl, config_json, config_hash)
       @image_full_name = image_full_name
-      @template_variables = template_variables
-
-      # image_full_name should be in the form of:
-      # redis:4.0.1
-      # kaspergrubbe/secretimage:latest
-      # billetto/billetto-rails:4.2.24
-      unless @image_full_name.match(/\A[a-zA-Z0-9\/\-\_]+\:[a-zA-Z0-9\.\-\_]+\z/)
-        raise Nomade::Job::FormattingError.new("Image-format wrong: #{@image_full_name}")
-      end
-
-      @config_hcl = render_erb(template_file)
-      @config_json = convert_job_hcl_to_json(@config_hcl)
-      @config_hash = JSON.parse(@config_json)
+      @config_hcl = config_hcl
+      @config_json = config_json
+      @config_hash = config_hash
     end
 
     def configuration(format)
@@ -28,17 +13,19 @@ module Nomade
         @config_hcl
       when :json
         @config_json
+      when :hash
+        @config_hash
       else
         @config_hash
       end
     end
 
     def job_name
-      @config_hash["Job"]["ID"]
+      @config_hash["ID"]
     end
 
     def job_type
-      @config_hash["Job"]["Type"]
+      @config_hash["Type"]
     end
 
     def image_name_and_version
@@ -53,25 +40,5 @@ module Nomade
       image_name_and_version.split(":").last
     end
 
-    def template_variables
-      @template_variables
-    end
-
-    private
-
-    def render_erb(erb_template)
-      file = File.open(erb_template).read
-      rendered = ERB.new(file, nil, '-').result(binding)
-
-      rendered
-    end
-
-    def convert_job_hcl_to_json(rendered_template)
-      exit_status, stdout, stderr = Shell.exec("nomad job run -output -no-color -", rendered_template)
-
-      JSON.pretty_generate({
-        "Job": JSON.parse(stdout)["Job"],
-      })
-    end
   end
 end

data/lib/nomade/job_builder.rb

@@ -0,0 +1,46 @@
+require "erb"
+require "json"
+
+module Nomade
+  class JobBuilder
+    def initialize(http)
+      @http = http
+    end
+
+    def build(template_file, image_full_name, template_variables = {})
+      # image_full_name should be in the form of:
+      # redis:4.0.1
+      # kaspergrubbe/secretimage:latest
+      # billetto/billetto-rails:4.2.24
+      unless image_full_name.match(/\A[a-zA-Z0-9\/\-\_]+\:[a-zA-Z0-9\.\-\_]+\z/)
+        raise Nomade::FormattingError.new("Image-format wrong: #{image_full_name}")
+      end
+
+      job_hcl = render_erb(template_file, image_full_name, template_variables)
+      job_json = @http.convert_hcl_to_json(job_hcl)
+      job_hash = JSON.parse(job_json)
+
+      Nomade::Job.new(image_full_name, job_hcl, job_json, job_hash)
+    end
+
+    private
+
+    def render_erb(erb_template, image_full_name, template_variables)
+      file = File.open(erb_template).read
+
+      local_binding = binding
+      local_binding.local_variable_set(:image_name_and_version, image_full_name)
+      local_binding.local_variable_set(:image_full_name, image_full_name)
+      local_binding.local_variable_set(:template_variables, template_variables)
+
+      # https://github.com/ruby/ruby/commit/3406c5d
+      rendered = if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(file, trim_mode: '-')
+      else
+        ERB.new(file, nil, '-')
+      end
+
+      rendered.result(local_binding)
+    end
+  end
+end

data/lib/nomade/logger.rb

@@ -10,8 +10,10 @@ module Nomade
       end
 
       Yell.new do |l|
-        l.adapter STDOUT, level: stdout
-        l.adapter STDERR, level: [:error, :fatal]
+        unless ARGV.include?("-q")
+          l.adapter STDOUT, level: stdout
+          l.adapter STDERR, level: [:error, :fatal]
+        end
       end
     end
   end

data/lib/nomade.rb

@@ -1,12 +1,13 @@
 ENV["TZ"] = "UTC"
 
-require "nomade/shell"
-require "nomade/job"
-require "nomade/logger"
+require "nomade/decorators"
+require "nomade/deployer"
 require "nomade/exceptions"
+require "nomade/hooks"
 require "nomade/http"
-require "nomade/deployer"
-require "nomade/decorators"
+require "nomade/job"
+require "nomade/job_builder"
+require "nomade/logger"
 
 module Nomade
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nomade
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
 platform: ruby
 authors:
 - Kasper Grubbe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-26 00:00:00.000000000 Z
+date: 2020-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yell
@@ -42,16 +42,44 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.18.5
+- !ruby/object:Gem::Dependency
+  name: irb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
 description: 
 email: nomade@kaspergrubbe.com
 executables: []
@@ -62,10 +90,11 @@ files:
 - lib/nomade/decorators.rb
 - lib/nomade/deployer.rb
 - lib/nomade/exceptions.rb
+- lib/nomade/hooks.rb
 - lib/nomade/http.rb
 - lib/nomade/job.rb
+- lib/nomade/job_builder.rb
 - lib/nomade/logger.rb
-- lib/nomade/shell.rb
 homepage: https://billetto.com
 licenses:
 - MIT

data/lib/nomade/shell.rb

@@ -1,45 +0,0 @@
-require "open3"
-
-module Nomade
-  class Shell
-    def self.exec(command, input = nil, allowed_exit_codes = [0])
-      Nomade.logger.debug("+: #{command}")
-
-      process, status, stdout, stderr = Open3.popen3(command) do |stdin, stdout, stderr, wait_thread|
-        if input
-          stdin.puts(input)
-        end
-        stdin.close
-
-        threads = {}.tap do |it|
-          it[:stdout] = Thread.new do
-            output = []
-            stdout.each do |l|
-              output << l
-              Nomade.logger.debug(l)
-            end
-            Thread.current[:output] = output.join
-          end
-
-          it[:stderr] = Thread.new do
-            output = []
-            stderr.each do |l|
-              output << l
-              Nomade.logger.debug(l)
-            end
-            Thread.current[:output] = output.join
-          end
-        end
-        threads.values.map(&:join)
-
-        [wait_thread.value, wait_thread.value.exitstatus, threads[:stdout][:output], threads[:stderr][:output]]
-      end
-
-      unless allowed_exit_codes.include?(status)
-        raise "`#{command}` failed with status=#{status}"
-      end
-
-      return [status, stdout, stderr]
-    end
-  end
-end