seiso-import_chef 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74d11fdc37b5d0cd7b62dcae3e6b6d558ab16268
-  data.tar.gz: 8cc930b7adc6d0f8982550976b6246163b336937
+  metadata.gz: 91aca0b492acb4adeb8285b1e17a60553914650b
+  data.tar.gz: 973f6a0a8514193fc0648d51d3a28b995c671588
 SHA512:
-  metadata.gz: e415076f0438c0a36d53e9519188a88c1074242c06c1f214e6936bf05352723b42282e87da0a9a34bdd8456ebcb63ebfa8bf7a6655580d2c7401534213ac29f9
-  data.tar.gz: f979b0e2bdd70f046af738526050aef59950fc01d93dc3f67b59668ee49b4f98b97f9b4a4da92176e8cf774a1f71e58d935048d59b60981ebad26e85e70815c3
+  metadata.gz: 3d3dfe57a41c2a6dce148b5c6808a662c5c1345590975a48f2f814c2ae074faa8b14259dadac3fea2ddd5565660216802fff2c3cfb4b86c1244eaf932fb08e7b
+  data.tar.gz: 84e2a404ab5b99253398abcaf67fe1ef817ef9cfdeaf4f8134d2d9c528c69507c802d470ea67b10b2c95f93a259b4ff291271cc7862f7c8e4bd0784beb5295f8

data/.travis.yml

@@ -1,3 +1,8 @@
 language: ruby
 rvm:
-  - 1.9.3
+  - 2.1.4
+before_install:
+  - gem uninstall bundler
+  - gem install bundler --version '1.10.5'
+#install:
+#  - bundle install

data/Rakefile

@@ -13,3 +13,8 @@ Rake::TestTask.new do |t|
   t.test_files = FileList[ 'test/test_*.rb' ]
   t.verbose = true
 end
+
+task :run do
+  Rake::Task['install'].execute
+  system('seiso-import-chef --chef ~/.seiso-importers/chef.yml --seiso ~/.seiso-importers/seiso3.yml')
+end

data/lib/seiso/import_chef.rb

@@ -1,7 +1,8 @@
-require "chef/node"
-require "chef/rest"
-require "seiso/connector"
+require "chef"
+require "json"
+
 require_relative "import_chef/chef_machine_mapper"
+require_relative "import_chef/connector"
 
 # Seiso namespace module
 module Seiso
@@ -12,15 +13,17 @@ module Seiso
   # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
   # License:: Apache 2.0
   class ImportChef
-    
-    # Batch size for putting nodes into Seiso.
+
     PAGE_SIZE = 20
-    
+
     def initialize(chef_settings, seiso_settings)
-      @chef = Chef::REST.new(
-        "#{chef_settings['base_uri']}",
-        "#{chef_settings['client_name']}",
-        "#{chef_settings['signing_key']}")
+
+      chef_options = {
+        :client_name => chef_settings['client_name'],
+        :signing_key_filename => chef_settings['signing_key'],
+        :inflate_json_class => true
+      }
+      @chef = Chef::ServerAPI.new(chef_settings['base_uri'], chef_options)        
       @seiso = Seiso::Connector.new seiso_settings
       @mapper = Seiso::ImportChef::ChefMachineMapper.new
     end
@@ -28,26 +31,23 @@ module Seiso
     def seiso
       @seiso
     end
-    
+
     # Imports all nodes from the Chef server into Seiso.
     def import_all
       nodes = @chef.get_rest "/nodes"
       node_names = nodes.keys.sort!
-      
-      slices = node_names.each_slice(PAGE_SIZE).to_a
       node_count = 0
       problem_nodes = []
+      machine_batch = []
+      slices = node_names.each_slice(PAGE_SIZE).to_a
       slices.each_with_index do |slice, page_index|
-        puts "Building page #{page_index + 1} (total_nodes=#{node_names.length}, page_size=#{PAGE_SIZE})"
-        page = []
+        machine_batch = []
         slice.each do |name|
-          # name is case-sensitive in Chef, so we can't just downcase it
           node_count += 1
           puts "Getting node #{node_count} of #{node_names.length}: #{name}"
           begin
             node = @chef.get_rest "/nodes/#{name}"
-            node_automatic = node.automatic
-            
+            node_automatic = node['automatic']
             # Have observed this. Need to decide whether to handle default/override/etc.
             if node_automatic.empty?
               puts "WARNING: Node #{name} has no automatic attributes"
@@ -55,15 +55,12 @@ module Seiso
                 "name" => name,
                 "reason" => "Node has no automatic attributes."
               }
+              # Since a node without automatic attributes means it doesn't have an IP,
+              # we don't want to import. [IDM]
               next
             end
-            
             machine = @mapper.map_one(node_automatic)
-            
-            # FIXME Just setting this here for now since I'm not sure which node field the name corresponds to.
-            # It's not exactly the FQDN (the case is different).
             machine['name'] = name
-            page << machine
           rescue Net::HTTPServerException
             puts "ERROR: Couldn't get #{name}"
             problem_nodes << {
@@ -71,16 +68,11 @@ module Seiso
               "reason" => "Couldn't get node from Chef server even though Chef server provided the name."
             }
           end
+          machine_batch << machine
         end
-        
-        # FIXME This is not detecting some HTTP 500s that are happening. When this happens, the whole page fails, the server
-        # returns an HTTP 500 (I think, anyway--need to confirm), and this call doesn't catch the error. Somehow we need to
-        # surface these as problem nodes too, maybe indicating that the server didn't like them, as opposed to not being able to
-        # read them from Chef server.
-        puts "Flushing nodes to Seiso"
-        seiso.post_items('machines', page)
+        seiso.send_data(machine_batch)
       end
-      
+
       if problem_nodes.empty?
         puts "Successfully imported all machines"
       else

data/lib/seiso/import_chef/connector.rb

@@ -0,0 +1,101 @@
+require 'json'
+require 'net/http'
+require 'openssl'
+
+module Seiso
+
+  # REST interface to Seiso API. 
+  # 
+  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+  # Copyright:: Copyright (c) 2014-2016
+  # License:: Apache 2.0
+  class Connector
+
+    def initialize(settings)
+      @username = settings['username']
+      @password = settings['password']
+      @use_ssl = settings['use_ssl']
+      @request_headers = {
+        "accept" => "application/json",
+        "Content-Type" => "application/json",
+      }
+      @base_uri = URI(settings['base_uri'])
+      @verify_mode = settings['ssl_verify_none'] ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+    end
+
+    def send_data(data)
+      Net::HTTP.start(
+        @base_uri.host, 
+        @base_uri.port,
+        :use_ssl => @use_ssl,
+        :verify_mode => @verify_mode) do |http|
+          http.keep_alive_timeout = 10
+          upsert_machines(http, data)
+      end
+    end
+
+    private
+
+    def get(http, path)
+      request = Net::HTTP::Get.new(path, initheader = @request_headers)
+      request.basic_auth(@username, @password)
+      response = http.request(request)
+      if response.kind_of?(Net::HTTPSuccess)
+        return JSON.parse(response.body)
+      elsif response.kind_of?(Net::HTTPNotFound)
+        return nil
+      else
+        # Not sure about what other conditions, if any, need to be checked here. [IDM]
+        return nil
+      end
+    end
+
+    def do_post(http, path, data)
+      puts "POST #{path}"
+      request = Net::HTTP::Post.new(path, initheader = @request_headers)
+      request.basic_auth(@username, @password)
+      request.body = data.to_json
+      response = http.request(request)
+      display_response(response)
+    end
+
+    def do_put(http, path, data)
+      puts "PUT #{path}"
+      request = Net::HTTP::Put.new(path, initheader = @request_headers)
+      request.basic_auth(@username, @password)
+      request.body = data.to_json
+      response = http.request(request)
+       display_response(response)
+    end
+
+    def upsert_machine(http, payload)
+      search_path = "#{@base_uri}/api/machines/search/findByName?name=#{payload['name']}"
+      machines_path = "#{@base_uri}/api/machines"
+      seiso_machine = get(http, search_path)
+      if seiso_machine.nil?
+        do_post(http, machines_path, payload)
+      else
+        self_url = seiso_machine['_links']['self']['href']
+        do_put(http, self_url, payload)
+      end
+    end
+
+    def upsert_machines(http, machines)
+      machines.each do |machine|
+        upsert_machine(http, machine)
+      end
+      puts "Machine page saved to Seiso."
+    end
+
+    # Displays an HTTP response on the command line.
+    # - response: response to display
+    def display_response(response)
+      case response.code
+      when "500"
+        puts "Server error: #{response.code} #{response.message}"
+      else
+        puts "#{response.code} #{response.message}"
+      end
+    end
+  end
+end

data/seiso-import_chef.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "seiso-import_chef"
-  spec.version       = "0.12.0"
+  spec.version       = "0.13.0"
   spec.authors       = ["Willie Wheeler"]
   spec.email         = ["wwheeler@expedia.com"]
   spec.summary       = "Imports Chef data into Seiso."
@@ -23,5 +23,4 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
 
   spec.add_runtime_dependency "chef", "~> 12.4"
-  spec.add_runtime_dependency "seiso-connector", "~> 0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seiso-import_chef
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Willie Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-02 00:00:00.000000000 Z
+date: 2016-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,20 +52,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '12.4'
-- !ruby/object:Gem::Dependency
-  name: seiso-connector
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
 description: Currently supports importing Chef nodes as Seiso machines.
 email:
 - wwheeler@expedia.com
@@ -83,6 +69,7 @@ files:
 - bin/seiso-import-chef
 - lib/seiso/import_chef.rb
 - lib/seiso/import_chef/chef_machine_mapper.rb
+- lib/seiso/import_chef/connector.rb
 - refresh
 - sample_conf/chef.yml.sample
 - sample_conf/seiso.yml.sample
@@ -108,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Imports Chef data into Seiso.