seiso-import_master 0.0.7 → 0.0.8

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZTRjZDQ3YzYzOWUyOTdjYWY2MTc0ZGIxM2E0YTI4OGQ3ZTAzOWRjNw==
+    ZTA3MTk0YTNlYzIxMDM3MzhlNTMwYWFhMGRiNzVhODU5MmYyZDc2Ng==
   data.tar.gz: !binary |-
-    NzNhOWRlNTNjMGQxZGI1ODhhMzExMzY0ZTdkOWQ4OGM4ZDdlM2RmNg==
+    ZGE0MjU4OGIxNWRlYjQyZGY1N2Y5NzMwYjZkOTRlNDg1YzUyOGQ2Mg==
 SHA512:
   metadata.gz: !binary |-
-    NTY3N2EzOWY2NGY0OGI0N2VmYmMwNTcyMjIwZDI1NDk2Y2MwMDI4NmExZTA4
-    NWQ5NjJiYjMyZjFmMzk1Y2ZmNmMyMTU1MTk3ZjA0N2YyZGIyNTY1Yzg0ZWRl
-    Y2M3ZTZlODJmZmRmMzBlNjYyNjkxNjAwZGFhZWE4NWQ5MDM1ZTE=
+    NWY0MGZkYjRkYzA1MTNlMDE3NzM1ZDBmM2RiZGE5NDZlY2JlYTY0Yzc2YWIz
+    YTliN2E4YjQ0ODYyMzY3MjFkNWE1ZDk0MGI2NjM4YzM1ODk5Yjc0MGRlMjU5
+    MmMxODkzYWZjNzVhNGY0ZmJhMTY0NWUyYTYxZTc0OWQzMzA4MGI=
   data.tar.gz: !binary |-
-    NTg4MDMxMWMyMjM1NmEwMjUyZWZjZTQ2MDEwYTc0YjM5NjkzMTc4M2VlYWFl
-    ZWI5ZWYxNGZlYTU2ZTMxNjNiYWE3YjdhNTZmMGNjMGEyMDE2ZTBhMzU0NGYx
-    NjI1NjJiYjc4NWY0ZDFlYjkzN2ExMDVkNzhjYjA5MDg4MTlkMmU=
+    ZjI2MjlhNzIzNWY2NTIzNmY3Y2VhMTlhYTI4ODE2NjRjYjM0MGRiZjRmMzRh
+    NTVlMTBmMDZmMjkwZjA0Y2NjMTgxOTJiM2M3ZDZhZWM0ZTQxMjdlYzI4NTI5
+    ZGRlNzJjOTAwZmUxMjliNzU4ZmI4ODhmYzllNGEzNzY0MGJjYTM=

data/lib/seiso/import_master.rb

@@ -1,7 +1,7 @@
 require "json"
 require "require_all"
-require "seiso/connector"
 require "yaml"
+require "seiso/connector"
 
 require_rel "import_master"
 
@@ -39,16 +39,24 @@ module Seiso
       rest_connector_v2 = rest_connector(seiso_settings, "application/hal+json")
 
       @simple_importer = Importers::SimpleImporter.new(basic_mapper, seiso, uri_factory_v1, rest_connector_v1)
+
+      # TODO Clean this up. Just use v2 URI factory and connector.
       @importers = {
         'nodes' => Importers::NodeImporter.new(node_mapper, uri_factory_v1, rest_connector_v1),
         'services' => Importers::ServiceImporter.new(
-            service_mapper,
             basic_mapper,
+            service_mapper,
             uri_factory_v1,
             uri_factory_v2,
             rest_connector_v1,
             rest_connector_v2),
-        'service-instances' => Importers::ServiceInstanceImporter.new(service_instance_mapper, seiso, uri_factory_v1, rest_connector_v1)
+        'service-instances' => Importers::ServiceInstanceImporter.new(
+            service_instance_mapper,
+            uri_factory_v1,
+            uri_factory_v2,
+            seiso,
+            rest_connector_v1,
+            rest_connector_v2)
       }
     end
 
@@ -81,7 +89,7 @@ module Seiso
       use_ssl = settings['use_ssl'] || false
       scheme = use_ssl ? "https" : "http"
       base_uri = "#{scheme}://#{host}:#{port}/v1"
-      Util::UriFactoryV1.new base_uri
+      Seiso::Connector::UriFactoryV1.new base_uri
     end
 
     def uri_factory_v2(settings)
@@ -90,7 +98,7 @@ module Seiso
       use_ssl = settings['use_ssl'] || false
       scheme = use_ssl ? "https" : "http"
       base_uri = "#{scheme}://#{host}:#{port}/v2"
-      Util::UriFactoryV2.new base_uri
+      Seiso::Connector::UriFactoryV2.new base_uri
     end
 
     def rest_connector(settings, media_type)
@@ -100,7 +108,7 @@ module Seiso
       ignore_cert = settings['ignore_cert'] || false
       username = settings['username']
       password = settings['password']
-      Util::RestConnector.new(host, port, use_ssl, ignore_cert, username, password, media_type)
+      Seiso::Connector::RestConnector.new(host, port, use_ssl, ignore_cert, username, password, media_type)
     end
   end
 end

data/lib/seiso/import_master/importers/service_importer.rb

@@ -4,6 +4,8 @@ module Seiso
 	class ImportMaster
 		module Importers
 
+      # TODO Clean this up. Just use v2 URI factory and connector.
+
 			# Imports a service document into Seiso.
 			# 
       # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
@@ -12,8 +14,8 @@ module Seiso
 			class ServiceImporter < BaseImporter
 
 				def initialize(
-            service_mapper,
             basic_mapper,
+            service_mapper,
             uri_factory_v1,
             uri_factory_v2,
             rest_connector_v1,
@@ -32,32 +34,38 @@ module Seiso
 					services = doc['items']
 					puts "Importing service document"
 
-					doc_links = detach_children(services, 'service', 'key', 'docLinks')
+					sdm_doc_links = detach_children(services, 'service', 'key', 'docLinks')
 
 					seiso_services = services.map { |s| @service_mapper.seiso_service s }
           services_uri = @uri_factory_v1.services_uri true
           @rest_connector_v1.post(services_uri, seiso_services)
 
-          import_doc_links doc_links
+          import_doc_links(services, sdm_doc_links)
         end
 
         private
 
-        def import_doc_links(doc_links)
-          doc_links_by_service = group_doc_links_by_service doc_links
+        # - services: services
+        # - doc_links: SDM doc links
+        def import_doc_links(services, doc_links)
 
-          doc_links_by_service.each do |service_key, doc_links|
+          # We can't simply infer the services from the doc links, because we have to handle the case
+          # where we remove all doc links from a service instance that formerly had some.
+
+          doc_links_by_service = group_doc_links_by_service doc_links
+          services.each do |service|
+            service_key = service['key']
+            service_doc_links = doc_links_by_service[service_key] || []
             doc_links_uri = @uri_factory_v2.doc_links_uri service_key
 
             # Get service's doc links so we can
             # 1) Delete stale doc links
             # 2) Avoid duplicate imports
-            seiso_doc_links_response = @rest_connector_v2.get(doc_links_uri)
-            seiso_doc_links = JSON.parse(seiso_doc_links_response.body)
-
-            to_delete = seiso_doc_links_to_delete(doc_links, seiso_doc_links)
+            seiso_doc_links_response = @rest_connector_v2.get doc_links_uri
+            seiso_doc_links = JSON.parse seiso_doc_links_response.body
 
             # Delete stale doc links
+            to_delete = seiso_doc_links_to_delete(service_doc_links, seiso_doc_links)
             to_delete.each do |seiso_doc_link|
               href = seiso_doc_link['_links']['self']['href']
               uri = URI.parse href
@@ -67,7 +75,7 @@ module Seiso
             # Classify remaining links as post or put
             to_post = []
             to_put = []
-            doc_links.each do |sdm_doc_link|
+            service_doc_links.each do |sdm_doc_link|
               sdm_title = sdm_doc_link['title']
               post_it = true
               seiso_doc_links.each do |seiso_doc_link|

data/lib/seiso/import_master/importers/service_instance_importer.rb

@@ -4,6 +4,10 @@ module Seiso
 	class ImportMaster
 		module Importers
 
+      # TODO Clean this up. Just use v2 URI factory and connector.
+      # Right now we're using three separate Seiso connectors (and two versions of the Seiso API)
+      # and this sucks. [WLW]
+
 			# Imports a service instance document into Seiso.
 			# 
       # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
@@ -11,30 +15,106 @@ module Seiso
       # License:: Apache 2.0
 			class ServiceInstanceImporter < BaseImporter
 
-				def initialize(mapper, seiso, uri_factory_v1, rest_connector_v1)
-					@mapper = mapper
-					@seiso = seiso
+				def initialize(
+            service_instance_mapper,
+            uri_factory_v1,
+            uri_factory_v2,
+            seiso,
+            rest_connector_v1,
+            rest_connector_v2)
+
+          @service_instance_mapper = service_instance_mapper
           @uri_factory_v1 = uri_factory_v1
+          @uri_factory_v2 = uri_factory_v2
+					@seiso = seiso
           @rest_connector_v1 = rest_connector_v1
+          @rest_connector_v2 = rest_connector_v2
 				end
 
 				# Imports the service instance document
 				def import(doc)
-					service_instances = doc['items']
-		      ports = detach_children(service_instances, 'serviceInstance', 'key', 'ports')
-		      roles = detach_children(service_instances, 'serviceInstance', 'key', 'ipAddressRoles')
+					sdm_service_instances = doc['items']
+		      sdm_ports = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'ports')
+		      sdm_roles = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'ipAddressRoles')
+          sdm_seyren_checks = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'seyrenChecks')
 
-		      seiso_service_instances = service_instances.map { |si| @mapper.seiso_service_instance si }
+          import_service_instances sdm_service_instances
+          import_ports sdm_ports
+          import_ip_address_roles sdm_roles
+          import_seyren_checks sdm_seyren_checks
+				end
+
+        private
+
+        def import_service_instances(sdm_service_instances)
+          seiso_service_instances = sdm_service_instances.map do |si|
+            @service_instance_mapper.seiso_service_instance si
+          end
           service_instances_uri = @uri_factory_v1.service_instances_uri true
           @rest_connector_v1.post(service_instances_uri, seiso_service_instances)
+        end
 
-		      seiso_ports = ports.map { |p| @mapper.seiso_service_instance_port p }
-		      @seiso.post_items('service-instance-ports', seiso_ports)
+        def import_ports(sdm_ports)
+          seiso_ports = sdm_ports.map do |p|
+            @service_instance_mapper.seiso_service_instance_port p
+          end
+          @seiso.post_items('service-instance-ports', seiso_ports)
+        end
 
-		      seiso_roles = roles.map { |r| @mapper.seiso_ip_address_role r }
-		      @seiso.post_items('ip-address-roles', seiso_roles)
-				end
-			end
-		end
-	end
-end
+        def import_ip_address_roles(sdm_roles)
+          seiso_roles = sdm_roles.map do |r|
+            @service_instance_mapper.seiso_ip_address_role r
+          end
+          @seiso.post_items('ip-address-roles', seiso_roles)
+        end
+
+        def import_seyren_checks(sdm_checks)
+          puts "Importing Seyren checks"
+
+          # Group checks by service instance
+          checks_by_service_instance = {}
+          sdm_checks.each do |c|
+            si_key = c['serviceInstance']
+            checks = checks_by_service_instance[si_key]
+            if checks.nil?
+              checks = []
+              checks_by_service_instance[si_key] = checks
+            end
+            checks << c
+          end
+
+          # Iterate over service instances, posting the checks for each one.
+          # TODO Need to handle stale checks and separate POST/PUT
+          checks_by_service_instance.each do |si_key, si_checks|
+            puts "Importing Seyren checks: serviceInstance=#{si_key}"
+
+            # checks_uri is the Seiso collection resource for posting checks to a service instance
+            checks_uri = @uri_factory_v2.service_instance_seyren_checks_uri si_key
+
+            # Build the list of Seiso Seyren check URIs that we want to post to checks_uri
+            uri_list = []
+            si_checks.each do |sdm_check|
+
+              # Look up the Seiso Seyren check URI, since we don't know it in advance.
+              # If it exists, add it to the URI list.
+              seyren_base_url = sdm_check['baseUrl']
+              seyren_id = sdm_check['id']
+              search_uri = @uri_factory_v2.find_seyren_check_uri(seyren_base_url, seyren_id)
+              response = @rest_connector_v2.get search_uri
+              if response.code == "200"
+                seiso_check = JSON.parse response.body
+                uri_list << seiso_check['_links']['self']['href']
+              end
+              # TODO Log a warning or something if the response code is 404 or other problem.
+            end
+
+            # Post the URI list to checks_uri
+            @rest_connector_v2.post_uri_list(checks_uri, uri_list)
+          end
+
+        end
+
+			end # class ServiceInstanceImporter
+		end # module Importers
+	end # class ImportMaster
+end # module Seiso

data/lib/seiso/import_master/mappers/master_item_mapper.rb

@@ -174,7 +174,7 @@ module Seiso
             }
           }
         end
-        
+                
         def status_type_mapper
           ->(st) {
             {

data/seiso-import_master.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "seiso-import_master"
-  spec.version       = "0.0.7"
+  spec.version       = "0.0.8"
   spec.authors       = ["Willie Wheeler"]
   spec.email         = ["wwheeler@expedia.com"]
   spec.summary       = "Imports Seiso data master files into Seiso."
@@ -22,6 +22,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 
-  spec.add_runtime_dependency "seiso-connector", "~> 0.1.0"
+  spec.add_runtime_dependency "seiso-connector", "~> 0.1.6"
   spec.add_runtime_dependency "require_all", "~> 1.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seiso-import_master
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Willie Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-20 00:00:00.000000000 Z
+date: 2015-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.0
+        version: 0.1.6
 - !ruby/object:Gem::Dependency
   name: require_all
   requirement: !ruby/object:Gem::Requirement
@@ -93,9 +93,6 @@ files:
 - lib/seiso/import_master/mappers/node_mapper.rb
 - lib/seiso/import_master/mappers/service_instance_mapper.rb
 - lib/seiso/import_master/mappers/service_mapper.rb
-- lib/seiso/import_master/util/rest_connector.rb
-- lib/seiso/import_master/util/uri_factory_v1.rb
-- lib/seiso/import_master/util/uri_factory_v2.rb
 - lib/seiso/import_master/validators/base_validator.rb
 - lib/seiso/import_master/validators/node_validator.rb
 - lib/seiso/import_master/validators/service_validator.rb

data/lib/seiso/import_master/util/rest_connector.rb

@@ -1,102 +0,0 @@
-require "json"
-require "net/http"
-require "openssl"
-
-module Seiso
-  class ImportMaster
-    module Util
-
-      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-      # License:: Apache 2.0
-      class RestConnector
-        HN_ACCEPT = "Accept"
-        HN_CONTENT_TYPE = "Content-Type"
-
-        def initialize(host, port, use_ssl, ignore_cert, username, password, media_type)
-          @host = host
-          @port = port
-          @use_ssl = use_ssl
-          @ignore_cert = ignore_cert
-          @username = username
-          @password = password
-          @media_type = media_type
-
-          # FIXME Not secure
-          @verify_mode = OpenSSL::SSL::VERIFY_NONE if @use_ssl
-        end
-
-        def get(uri)
-          request = Net::HTTP::Get.new uri.request_uri
-          request[HN_ACCEPT] = @media_type
-          request.basic_auth(@username, @password)
-          do_request(http, request)
-        end
-
-        def post(uri, resource)
-          request = Net::HTTP::Post.new uri.request_uri
-          request[HN_CONTENT_TYPE] = @media_type
-          request.basic_auth(@username, @password)
-          request.body = resource.to_json
-          do_request(http, request)
-        end
-
-        # Puts all resources using a keepalive connection
-        def put_all(uri_resource_pairs)
-          Net::HTTP.start(
-              @host,
-              @port,
-              :use_ssl => @use_ssl,
-              :verify_mode => @verify_mode) do |http|
-
-            uri_resource_pairs.each do |pair|
-              pair_uri = pair['uri']
-              pair_resource = pair['resource']
-              request = Net::HTTP::Put.new pair_uri.request_uri
-              request[HN_CONTENT_TYPE] = @media_type
-              request.basic_auth(@username, @password)
-              request.body = pair_resource.to_json
-              do_request(http, request)
-            end
-          end
-        end
-
-        def put(uri, resource)
-          request = Net::HTTP::Put.new uri.request_uri
-          request[HN_CONTENT_TYPE] = @media_type
-          request.basic_auth(@username, @password)
-          request.body = resource.to_json
-          do_request(http, request)
-        end
-
-        def delete(uri)
-          puts "DELETE #{uri}"
-          request = Net::HTTP::Delete.new uri.request_uri
-          request.basic_auth(@username, @password)
-          do_request(http, request)
-        end
-
-        private
-
-        def http
-          # Not sure if I can reuse this, so for now I'm just creating a new one every time.
-          http = Net::HTTP.new(@host, @port)
-          http.use_ssl = @use_ssl
-          http.verify_mode = @verify_mode
-          http
-        end
-
-        def do_request(http, request)
-          response = http.request request
-
-          if response.code.start_with?("4") || response.code.start_with?("5")
-            details = "(#{JSON.parse response.body})"
-          end
-          puts "#{request.method} #{request.path} => #{response.code} #{response.message} #{details}"
-          response
-        end
-
-      end # class RestConnector
-    end # module Util
-  end # class ImportMaster
-end # module Seiso

data/lib/seiso/import_master/util/uri_factory_v1.rb

@@ -1,61 +0,0 @@
-require "uri"
-
-module Seiso
-  class ImportMaster
-    module Util
-
-      # Seiso API v1 URI factory.
-      # 
-      # URI templating (embodied here) is a temporary strategy for obtaining URIs. The plan is
-      # to adopt something like a Traverson-style strategy.
-      # 
-      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-      # License:: Apache 2.0
-      class UriFactoryV1
-
-        def initialize(base_uri)
-          @base_uri = base_uri
-        end
-
-        def nodes_uri(batch)
-          if batch
-            uri "/nodes?mode=batch"
-          else
-            uri "/nodes"
-          end
-        end
-
-        def node_uri(name)
-          uri "/nodes/#{name}"
-        end
-        
-        def node_ip_address_uri(node_name, ip_address)
-          uri "/nodes/#{node_name}/ip-addresses/#{ip_address}"
-        end
-
-        def services_uri(batch)
-          if batch
-            uri "/services?mode=batch"
-          else
-            uri "/services"
-          end
-        end
-
-        def service_instances_uri(batch)
-          if batch
-            uri "/service-instances?mode=batch"
-          else
-            uri "/service-instances"
-          end
-        end
-
-        private
-
-        def uri(path)
-          URI.parse "#{@base_uri}#{path}"
-        end
-      end
-    end
-  end
-end

data/lib/seiso/import_master/util/uri_factory_v2.rb

@@ -1,34 +0,0 @@
-require "uri"
-
-module Seiso
-  class ImportMaster
-    module Util
-
-      # Seiso API v2 URI factory.
-      # 
-      # URI templating (embodied here) is a temporary strategy for obtaining URIs. The plan is
-      # to adopt something like a Traverson-style strategy.
-      # 
-      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-      # License:: Apache 2.0
-      class UriFactoryV2
-
-        def initialize(base_uri)
-          @base_uri = base_uri
-        end
-
-        def doc_links_uri(service_key)
-          uri "/services/#{service_key}/doc-links"
-        end
-
-        private
-
-        def uri(path)
-          URI.parse "#{@base_uri}#{path}"
-        end
-        
-      end # class UriFactoryV2
-    end # module Util
-  end # class ImportMaster
-end # module Seiso