seiso-import_master 0.0.12 → 3.0.0.M1

data/lib/seiso/import_master/importers/rotation_status_importer.rb

@@ -0,0 +1,25 @@
+class Seiso::ImportMaster
+  class Importers::RotationStatusImporter < Importers::BaseImporter
+
+    def initialize(api, rest_util, resolver)
+      self.api = api
+      self.rest_util = rest_util
+      self.mapper = Mappers::RotationStatusMapper.new resolver
+      self.repo_resource = api.rotationStatuses.get
+
+      @search_resource = repo_resource.search.get
+    end
+
+    def import(doc)
+      import_items(doc['items'], {})
+    end
+
+    def to_search_params(doc_rs, context)
+      { 'key' => doc_rs['key'] }
+    end
+
+    def find_resource(search_params)
+      @search_resource.findByKey(key: search_params['key']).get
+    end
+  end
+end

data/lib/seiso/import_master/importers/service_group_importer.rb

@@ -0,0 +1,25 @@
+class Seiso::ImportMaster
+  class Importers::ServiceGroupImporter < Importers::BaseImporter
+
+    def initialize(api, rest_util, resolver)
+      self.api = api
+      self.rest_util = rest_util
+      self.mapper = Mappers::ServiceGroupMapper.new resolver
+      self.repo_resource = api.serviceGroups.get
+
+      @search_resource = repo_resource.search.get
+    end
+
+    def import(doc)
+      import_items(doc['items'], {})
+    end
+
+    def to_search_params(doc_sg, context)
+      { 'key' => doc_sg['key'] }
+    end
+
+    def find_resource(search_params)
+      @search_resource.findByKey(key: search_params['key']).get
+    end
+  end
+end

data/lib/seiso/import_master/importers/service_importer.rb

@@ -1,143 +1,38 @@
-require_relative "base_importer"
-
-module Seiso
-	class ImportMaster
-		module Importers
-
-      # TODO Clean this up. Just use v2 URI factory and connector.
-
-			# Imports a service document into Seiso.
-			# 
-      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-      # License:: Apache 2.0
-			class ServiceImporter < BaseImporter
-
-				def initialize(
-            basic_mapper,
-            service_mapper,
-            uri_factory_v1,
-            uri_factory_v2,
-            rest_connector_v1,
-            rest_connector_v2)
-
-					@service_mapper = service_mapper
-          @basic_mapper = basic_mapper
-          @uri_factory_v1 = uri_factory_v1
-          @uri_factory_v2 = uri_factory_v2
-          @rest_connector_v1 = rest_connector_v1
-          @rest_connector_v2 = rest_connector_v2
-				end
-
-				# Imports the service document
-				def import(doc)
-					services = doc['items']
-					puts "Importing service document"
-
-					sdm_doc_links = detach_children(services, 'service', 'key', 'docLinks')
-
-					seiso_services = services.map { |s| @service_mapper.seiso_service s }
-          services_uri = @uri_factory_v1.services_uri true
-          @rest_connector_v1.post(services_uri, seiso_services)
-
-          import_doc_links(services, sdm_doc_links)
-        end
-
-        private
-
-        # - services: services
-        # - doc_links: SDM doc links
-        def import_doc_links(services, doc_links)
-
-          # We can't simply infer the services from the doc links, because we have to handle the case
-          # where we remove all doc links from a service instance that formerly had some.
-
-          doc_links_by_service = group_doc_links_by_service doc_links
-          services.each do |service|
-            service_key = service['key']
-            service_doc_links = doc_links_by_service[service_key] || []
-            doc_links_uri = @uri_factory_v2.doc_links_uri service_key
-
-            # Get service's doc links so we can
-            # 1) Delete stale doc links
-            # 2) Avoid duplicate imports
-            seiso_doc_links_response = @rest_connector_v2.get doc_links_uri
-            seiso_doc_links = JSON.parse seiso_doc_links_response.body
-
-            # Delete stale doc links
-            to_delete = seiso_doc_links_to_delete(service_doc_links, seiso_doc_links)
-            to_delete.each do |seiso_doc_link|
-              href = seiso_doc_link['_links']['self']['href']
-              uri = URI.parse href
-              @rest_connector_v2.delete(uri)
-            end
-
-            # Classify remaining links as post or put
-            to_post = []
-            to_put = []
-            service_doc_links.each do |sdm_doc_link|
-              sdm_title = sdm_doc_link['title']
-              post_it = true
-              seiso_doc_links.each do |seiso_doc_link|
-                seiso_title = seiso_doc_link['title']
-                if sdm_title == seiso_title
-                  href = seiso_doc_link['_links']['self']['href']
-                  uri = URI.parse href
-                  new_seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
-                  to_put << {
-                    "uri" => uri,
-                    "resource" => new_seiso_doc_link
-                  }
-                  post_it = false
-                  break
-                end
-              end
-              to_post << sdm_doc_link if post_it
-            end
-
-            # Post new links
-            to_post.each do |sdm_doc_link|
-              seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
-              @rest_connector_v2.post(doc_links_uri, seiso_doc_link)
-            end
-
-            # Put existing links
-            @rest_connector_v2.put_all(to_put)
-          end
-        end
-
-        def group_doc_links_by_service(doc_links)
-          doc_links_by_service = {}
-          doc_links.each do |dl|
-            service_key = dl['service']
-            service_doc_links = doc_links_by_service[service_key]
-            if service_doc_links.nil?
-              service_doc_links = []
-              doc_links_by_service[service_key] = service_doc_links
-            end
-            service_doc_links << dl
-          end
-          doc_links_by_service
-        end
-
-        def seiso_doc_links_to_delete(sdm_doc_links, seiso_doc_links)
-          to_delete = []
-          seiso_doc_links.each do |seiso_doc_link|
-            seiso_title = seiso_doc_link['title']
-            delete_it = true
-            sdm_doc_links.each do |sdm_doc_link|
-              sdm_title = sdm_doc_link['title']
-              if sdm_title == seiso_title
-                delete_it = false
-                break
-              end
-            end
-            to_delete << seiso_doc_link if delete_it
-          end
-          to_delete
-        end
-
-			end # class ServiceImporter
-		end # module Importers
-	end # class ImportMaster
-end # module Seiso
+class Seiso::ImportMaster
+  # Imports a service document into Seiso.
+  #
+  # Author:: Willie Wheeler
+  # Author:: Ian McCunn
+  # Copyright:: Copyright (c) 2014-2016 Expedia, Inc.
+  # License:: Apache 2.0
+  class Importers::ServiceImporter < Importers::BaseImporter
+
+    def initialize(api, rest_util, resolver)
+      self.api = api
+      self.rest_util = rest_util
+      self.mapper = Mappers::ServiceMapper.new resolver
+      self.repo_resource = api.services.get
+
+      @validator = Validators::ServiceValidator.new
+      @search_resource = repo_resource.search.get
+    end
+
+    def import(doc)
+      @validator.validate doc
+      import_items(doc['items'], {})
+    end
+
+    # Can't make these protected, because that prevents BaseImporter from seeing them.
+#    protected
+
+    # BaseImporter callback
+    def to_search_params(doc_service, context)
+      { 'key' => doc_service['key'] }
+    end
+
+    # BaseImporter callback
+    def find_resource(search_params)
+      @search_resource.findByKey(key: search_params['key']).get
+    end
+  end
+end

data/lib/seiso/import_master/importers/service_instance_importer.rb

@@ -1,213 +1,60 @@
-require "uri"
-require_relative "base_importer"
-
-module Seiso
-	class ImportMaster
-		module Importers
-
-      # TODO Clean this up. Just use v2 URI factory and connector.
-      # Right now we're using three separate Seiso connectors (and two versions of the Seiso API)
-      # and this sucks. [WLW]
-
-			# Imports a service instance document into Seiso.
-			# 
-      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-      # License:: Apache 2.0
-			class ServiceInstanceImporter < BaseImporter
-
-				def initialize(
-            service_instance_mapper,
-            uri_factory_v1,
-            uri_factory_v2,
-            seiso,
-            rest_connector_v1,
-            rest_connector_v2)
-
-          @service_instance_mapper = service_instance_mapper
-          @uri_factory_v1 = uri_factory_v1
-          @uri_factory_v2 = uri_factory_v2
-					@seiso = seiso
-          @rest_connector_v1 = rest_connector_v1
-          @rest_connector_v2 = rest_connector_v2
-				end
-
-				# Imports the service instance document
-				def import(doc)
-					sdm_service_instances = doc['items']
-		      sdm_ports = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'ports')
-		      sdm_roles = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'ipAddressRoles')
-          sdm_dependencies = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'dependencies')
-          sdm_seyren_checks = detach_children(sdm_service_instances, 'serviceInstance', 'key', 'seyrenChecks')
-
-          import_service_instances sdm_service_instances
-          import_ports sdm_ports
-          import_ip_address_roles sdm_roles
-          import_dependencies sdm_dependencies
-          import_seyren_checks sdm_seyren_checks
-				end
-
-        private
-
-        def import_service_instances(sdm_service_instances)
-          seiso_service_instances = sdm_service_instances.map do |si|
-            @service_instance_mapper.seiso_service_instance si
-          end
-          service_instances_uri = @uri_factory_v1.service_instances_uri true
-          @rest_connector_v1.post(service_instances_uri, seiso_service_instances)
-        end
-
-        def import_ports(sdm_ports)
-          seiso_ports = sdm_ports.map do |p|
-            @service_instance_mapper.seiso_service_instance_port p
-          end
-          @seiso.post_items('service-instance-ports', seiso_ports)
-        end
-
-        def import_ip_address_roles(sdm_roles)
-          seiso_roles = sdm_roles.map do |r|
-            @service_instance_mapper.seiso_ip_address_role r
-          end
-          @seiso.post_items('ip-address-roles', seiso_roles)
-        end
-
-        def import_dependencies(sdm_dependencies)
-          puts "Importing dependencies"
-          seiso_dependencies = @service_instance_mapper.seiso_service_instance_dependencies sdm_dependencies
-          grouped_seiso_dependencies = group_dependencies_by_service_instance seiso_dependencies
-
-          # Iterate over the groups, deleting stale dependencies and importing the rest
-          grouped_seiso_dependencies.each do |key, group|
-            puts "key=#{key}, group=#{group}"
-            delete_stale_dependencies(key, group)
-
-            # Import the dependencies.
-            group.each do |d|
-              dependent_key = d['dependent']['key']
-              dependency_key = d['dependency']['key']
-              puts "Importing dependency: dependent=#{dependent_key}, dependency=#{dependency_key}"
-              search_uri = @uri_factory_v2.uri "/service-instance-dependencies/search/find-by-keys?dependent=#{dependent_key}&dependency=#{dependency_key}"
-
-              # TODO 404 means POST, 200 means PUT
-              seiso_sid_response = @rest_connector_v2.get search_uri
-
-              if seiso_sid_response.code == "200"
-                # Found it, so PUT
-                seiso_sid = JSON.parse seiso_sid_response.body
-                dep_uri = URI.parse(seiso_sid['_links']['self']['href'])
-#                puts "dep_uri=#{dep_uri}"
-#                puts JSON.pretty_generate(d)
-                @rest_connector_v2.put(dep_uri, d)
-              elsif seiso_sid_response.code == "404"
-                # Not found, so POST
-                deps_uri = @uri_factory_v2.uri "/service-instance-dependencies"
-                @rest_connector_v2.post(deps_uri, d)
-              else
-                puts "Don't know how to handle response code #{seiso_sid.code}; skipping."
-              end
-            end
-          end
-        end
-
-        # Helper for import_dependencies
-        def group_dependencies_by_service_instance(seiso_dependencies)
-          grouped_seiso_dependencies = {}
-          seiso_dependencies.each do |d|
-            dependent_key = d['dependent']['key']
-            puts "dependent_key=#{dependent_key}"
-            group = grouped_seiso_dependencies[dependent_key]
-            if group.nil?
-              group = []
-              grouped_seiso_dependencies[dependent_key] = group
-            end
-            group << d
-          end
-          grouped_seiso_dependencies
-        end
-
-        # Helper for import_dependencies
-        # 
-        # - dependent_key: Key of the dependent service instance.
-        # - local_dependencies: List of local dependencies (in Seiso format) associated with
-        #   the specified dependent.
-        def delete_stale_dependencies(dependent_key, local_dependencies)
-          puts "Deleting stale dependencies for service instance #{dependent_key}"
-
-          local_dependency_keys = local_dependencies.map do |local|
-            # TODO Will need to update when I update the v2 API
-            local['dependency']['key']
-          end
-
-          # FIXME This isn't accounting for pagination!
-          # If there are a lot of remote dependencies, some may not be marked as stale.
-          # Just need to implement it. [WLW]
-          remote_dependencies_uri =
-              @uri_factory_v2.uri "/service-instance-dependencies/search/find-by-dependent?key=#{dependent_key}"
-          remote_dependencies_response = @rest_connector_v2.get remote_dependencies_uri
-          remote_dependencies = JSON.parse(remote_dependencies_response.body)['_embedded']['items']
-
-          # Identify stale dependencies
-          stale_remote_dependencies = remote_dependencies.reject do |remote|
-            local_dependency_keys.include? remote['_embedded']['dependency']['key']
-          end
-
-          # Delete stale dependencies from Seiso
-          if stale_remote_dependencies.empty?
-            puts "No stale dependencies for service instance #{dependent_key}"
-          else
-            stale_remote_dependencies.each do |stale|
-              uri = URI.parse stale['_links']['self']['href']
-              @rest_connector_v2.delete uri
-            end
-          end
-        end
-
-        def import_seyren_checks(sdm_checks)
-          puts "Importing Seyren checks"
-
-          # Group checks by service instance
-          checks_by_service_instance = {}
-          sdm_checks.each do |c|
-            si_key = c['serviceInstance']
-            checks = checks_by_service_instance[si_key]
-            if checks.nil?
-              checks = []
-              checks_by_service_instance[si_key] = checks
-            end
-            checks << c
-          end
-
-          # Iterate over service instances, posting the checks for each one.
-          # TODO Need to handle stale checks and separate POST/PUT
-          checks_by_service_instance.each do |si_key, si_checks|
-            puts "Importing Seyren checks: serviceInstance=#{si_key}"
-
-            # checks_uri is the Seiso collection resource for posting checks to a service instance
-            checks_uri = @uri_factory_v2.service_instance_seyren_checks_uri si_key
-
-            # Build the list of Seiso Seyren check URIs that we want to post to checks_uri
-            uri_list = []
-            si_checks.each do |sdm_check|
-
-              # Look up the Seiso Seyren check URI, since we don't know it in advance.
-              # If it exists, add it to the URI list.
-              seyren_base_url = sdm_check['baseUrl']
-              seyren_id = sdm_check['id']
-              search_uri = @uri_factory_v2.find_seyren_check_uri(seyren_base_url, seyren_id)
-              response = @rest_connector_v2.get search_uri
-              if response.code == "200"
-                seiso_check = JSON.parse response.body
-                uri_list << seiso_check['_links']['self']['href']
-              end
-              # TODO Log a warning or something if the response code is 404 or other problem.
-            end
-
-            # Post the URI list to checks_uri
-            @rest_connector_v2.post_uri_list(checks_uri, uri_list)
-          end
-
-        end
-			end # class ServiceInstanceImporter
-		end # module Importers
-	end # class ImportMaster
-end # module Seiso
+class Seiso::ImportMaster
+  # Imports a service instance document into Seiso.
+  #
+  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+  # Author:: Chris Brown
+  # Copyright:: Copyright (c) 2014-2016 Expedia, Inc.
+  # License:: Apache 2.0
+  class Importers::ServiceInstanceImporter < Importers::BaseImporter
+
+    def initialize(api, rest_util, resolver)
+      self.api = api
+      self.rest_util = rest_util
+      self.mapper = Mappers::ServiceInstanceMapper.new resolver
+      self.repo_resource = api.serviceInstances.get
+
+      @ipr_importer = Importers::IpAddressRoleImporter.new(api, rest_util)
+      @sip_importer = Importers::ServiceInstancePortImporter.new(api, rest_util)
+      @search_resource = repo_resource.search.get
+    end
+
+    def import(doc)
+      import_items(doc['items'], {})
+    end
+
+    # Can't make these protected, because that prevents BaseImporter from seeing them.
+#    protected
+
+    # BaseImporter callback
+    def to_search_params(doc_si, context)
+      { 'key' => doc_si['key'] }
+    end
+
+    # BaseImporter callback
+    def find_resource(search_params)
+      @search_resource.findByKey(key: search_params['key']).get
+    end
+
+    # BaseImporter callback
+    def context_for(parent_resource, parent_context)
+      { 'serviceInstance' => parent_resource }
+    end
+
+    # BaseImporter callback
+    def import_children_of(doc_si, context)
+
+      # Create a default IP address role if one doesn't exist. Its name is "default".
+      doc_iprs = doc_si['ipAddressRoles']
+      if doc_iprs.nil? || doc_iprs.empty?
+        doc_iprs = [ { 'name' => 'default', 'description' => 'Default role' } ]
+      end
+
+      # This doesn't have to be defined. For example, agents won't have ports.
+      # If it's not defined, we don't need to create an empty list over it.
+      doc_sips = doc_si['ports']
+
+      @ipr_importer.import_items(doc_iprs, context)
+      @sip_importer.import_items(doc_sips, context) unless doc_sips.nil?
+    end
+  end
+end