seiso-import_master 0.0.6 → 0.0.7

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjdmMzQyNjc3ZTI5YTdmNGM4MGUxZDA4NzYwOTU4YzhmNjBkY2RlYQ==
+    ZTRjZDQ3YzYzOWUyOTdjYWY2MTc0ZGIxM2E0YTI4OGQ3ZTAzOWRjNw==
   data.tar.gz: !binary |-
-    ZTgzMjBmOTNhMjhmZDA3ZmYzYzUzNTdmNTQzZGI4MThmOWNlMjc5Mg==
+    NzNhOWRlNTNjMGQxZGI1ODhhMzExMzY0ZTdkOWQ4OGM4ZDdlM2RmNg==
 SHA512:
   metadata.gz: !binary |-
-    ZThiZGYwODM2ZmQyNGUwNjcxYThlNWJiNjE2ZDhmODE1MzJiMDcwYjU3YmEx
-    MWQ5OGY0NTY0NDE0Mjg1ZTBjMzUyMTRiMjllZmZiYzM0MTRlMmU4OTQzZmM5
-    ZTkwMmNhYTU4OWU1MjdiMWJjZjc3YzU1ZDY3YWI2NjU0ZWVjNDc=
+    NTY3N2EzOWY2NGY0OGI0N2VmYmMwNTcyMjIwZDI1NDk2Y2MwMDI4NmExZTA4
+    NWQ5NjJiYjMyZjFmMzk1Y2ZmNmMyMTU1MTk3ZjA0N2YyZGIyNTY1Yzg0ZWRl
+    Y2M3ZTZlODJmZmRmMzBlNjYyNjkxNjAwZGFhZWE4NWQ5MDM1ZTE=
   data.tar.gz: !binary |-
-    ZDk2YTQxZmFlZGU3MjFmYTQ1MTlhMzA0YTNhMmQyZjk3Mjg0YjM0ZWZmMTQy
-    N2IzMDU1ZmE1YzgxMTZjZjg1ZDE2MGU5MWMyYmZjNDQ2YWE3Y2I5YTU4NjE3
-    OWI2MjJhNmQxYmQ3Y2E3NGFlNjY2MDk1MDZkMGRiM2FhN2NjZGE=
+    NTg4MDMxMWMyMjM1NmEwMjUyZWZjZTQ2MDEwYTc0YjM5NjkzMTc4M2VlYWFl
+    ZWI5ZWYxNGZlYTU2ZTMxNjNiYWE3YjdhNTZmMGNjMGEyMDE2ZTBhMzU0NGYx
+    NjI1NjJiYjc4NWY0ZDFlYjkzN2ExMDVkNzhjYjA5MDg4MTlkMmU=

data/README.md

@@ -39,3 +39,17 @@ Or install it yourself as:
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new pull request.
+
+## Development
+
+Rake tasks available via
+
+    $ rake tasks
+
+In particular, run the unit tests using
+
+    $ rake test
+
+Build and install the gem using Rake:
+
+    $ rake install

data/Rakefile

@@ -5,11 +5,13 @@ task :default => [ :test ]
 
 task :tasks do
   puts "Tasks:"
-  puts "test : Run unit tests"
+  puts "install : Build and install seiso-import_master gem"
+  puts "test    : Run unit tests"
 end
 
 Rake::TestTask.new do |t|
   t.libs << 'test'
   t.test_files = FileList[ 'test/test_*.rb' ]
   t.verbose = true
+  t.warning = true
 end

data/lib/seiso/import_master.rb

@@ -1,9 +1,10 @@
 require "json"
+require "require_all"
 require "seiso/connector"
 require "yaml"
-require_relative "import_master/master_item_mapper"
 
-# Seiso namespace module
+require_rel "import_master"
+
 module Seiso
 
   # Imports Seiso data master files into Seiso.
@@ -15,31 +16,47 @@ module Seiso
     
     # Initializes the importer with a Seiso connector.
     def initialize(seiso_settings)
-      @seiso = Seiso::Connector.new(seiso_settings)
-      @mapper = Seiso::ImportMaster::MasterItemMapper.new
       @loaders = {
         'json' => ->(file) { JSON.parse(IO.read(file)) },
         'yaml' => ->(file) { YAML.load_file file }
       }
-    end
 
-    def seiso
-      @seiso
-    end
+      # TODO Add other validators
+      @validators = {
+        'nodes' => Validators::NodeValidator.new,
+        'services' => Validators::ServiceValidator.new
+      }
 
-    def mapper
-      @mapper
-    end
+      basic_mapper = Mappers::MasterItemMapper.new
+      node_mapper = Mappers::NodeMapper.new
+      service_mapper = Mappers::ServiceMapper.new
+      service_instance_mapper = Mappers::ServiceInstanceMapper.new
+
+      seiso = Seiso::Connector.new seiso_settings
+      uri_factory_v1 = uri_factory_v1 seiso_settings
+      uri_factory_v2 = uri_factory_v2 seiso_settings
+      rest_connector_v1 = rest_connector(seiso_settings, "application/json")
+      rest_connector_v2 = rest_connector(seiso_settings, "application/hal+json")
 
-    def loaders
-      @loaders
+      @simple_importer = Importers::SimpleImporter.new(basic_mapper, seiso, uri_factory_v1, rest_connector_v1)
+      @importers = {
+        'nodes' => Importers::NodeImporter.new(node_mapper, uri_factory_v1, rest_connector_v1),
+        'services' => Importers::ServiceImporter.new(
+            service_mapper,
+            basic_mapper,
+            uri_factory_v1,
+            uri_factory_v2,
+            rest_connector_v1,
+            rest_connector_v2),
+        'service-instances' => Importers::ServiceInstanceImporter.new(service_instance_mapper, seiso, uri_factory_v1, rest_connector_v1)
+      }
     end
-    
+
     # Imports a list of master files in order. Legal formats are 'json' (default) and 'yaml'.
     def import_files(files, format = 'json')
       loop do
         file = files.pop
-        puts "Processing #{file}"
+        puts "Processing: #{file}"
         import_file(file, format)
         break if files.empty?
       end
@@ -47,74 +64,43 @@ module Seiso
     
     # Imports a data master file. Legal formats are 'json' (default) and 'yaml'.
     def import_file(file, format = 'json')
-      loader = loaders[format]
+      loader = @loaders[format]
       raise ArgumentError, "Illegal format: #{format}" if loader.nil?
       doc = loader.call(file)
-      import_doc doc
-    end
-    
-    # Imports a data master document.
-    def import_doc(doc)
       type = doc['type']
-      master_items = doc['items']
-      
-      # There are some special cases
-      if type == 'nodes'
-        si_key = doc['serviceInstance']
-        do_import_nodes(si_key, master_items)
-      elsif type == 'services'
-        do_import_services master_items
-      elsif type == 'service-instances'
-        do_import_service_instances master_items
-      else
-        do_import_items(type, master_items)
-      end
+      validator = @validators[type]
+      validator.validate(doc) unless validator.nil?
+      (@importers[type] || @simple_importer).import doc
     end
-    
+
     private
-    
-    def do_import_items(type, items)
-      seiso_items = mapper.map_all(type, items)
-      seiso.post_items(type, seiso_items)
-    end
-    
-    # Imports the nodes, along with their associated IP addresses.
-    def do_import_nodes(si_key, nodes)
-      nips = detach_children(nodes, 'node', 'name', 'ipAddresses')
-      nodes.each do |n|
-        n['serviceInstance'] = si_key
-      end
-      do_import_items('nodes', nodes)
-      do_import_items('node-ip-addresses', nips)
-    end
-    
-    def do_import_services(services)
-      doc_links = detach_children(services, 'service', 'key', 'docLinks')
-      do_import_items('services', services)
-#      do_import_items('doc-links', docLinks)
-      doc_links.each { |dl| puts dl.to_json }
+
+    def uri_factory_v1(settings)
+      host = settings['host']
+      port = settings['port']
+      use_ssl = settings['use_ssl'] || false
+      scheme = use_ssl ? "https" : "http"
+      base_uri = "#{scheme}://#{host}:#{port}/v1"
+      Util::UriFactoryV1.new base_uri
     end
 
-    # Imports the service instances, along with their associated ports and IP address roles.
-    def do_import_service_instances(service_instances)
-      ports = detach_children(service_instances, 'serviceInstance', 'key', 'ports')
-      roles = detach_children(service_instances, 'serviceInstance', 'key', 'ipAddressRoles')
-      do_import_items('service-instances', service_instances)
-      do_import_items('service-instance-ports', ports)
-      do_import_items('ip-address-roles', roles)
+    def uri_factory_v2(settings)
+      host = settings['host']
+      port = settings['port']
+      use_ssl = settings['use_ssl'] || false
+      scheme = use_ssl ? "https" : "http"
+      base_uri = "#{scheme}://#{host}:#{port}/v2"
+      Util::UriFactoryV2.new base_uri
     end
-    
-    # Enriches children with a link to the parent, detaches them from the parent, and returns all detached children.
-    def detach_children(parents, parent_prop, parent_key, child_prop)
-      all_children = []
-      parents.each do |p|
-        children = p[child_prop]
-        next if children.nil?
-        children.each { |c| c[parent_prop] = p[parent_key] }
-        all_children.push(*children)
-        p.delete child_prop
-      end
-      all_children
+
+    def rest_connector(settings, media_type)
+      host = settings['host']
+      port = settings['port']
+      use_ssl = settings['use_ssl'] || false
+      ignore_cert = settings['ignore_cert'] || false
+      username = settings['username']
+      password = settings['password']
+      Util::RestConnector.new(host, port, use_ssl, ignore_cert, username, password, media_type)
     end
   end
 end

data/lib/seiso/import_master/errors/invalid_document_error.rb

@@ -0,0 +1,11 @@
+module Seiso
+	class ImportMaster
+		module Errors
+      
+      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+      # License:: Apache 2.0
+			class InvalidDocumentError < StandardError; end
+		end
+	end
+end

data/lib/seiso/import_master/importers/base_importer.rb

@@ -0,0 +1,26 @@
+module Seiso
+	class ImportMaster
+		module Importers
+
+		  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+		  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+		  # License:: Apache 2.0
+			class BaseImporter
+
+	      # Enriches children with a link to the parent, detaches them from the parent, and returns
+	      # all detached children.
+	      def detach_children(parents, parent_prop, parent_key, child_prop)
+	        all_children = []
+	        parents.each do |p|
+	          children = p[child_prop]
+	          next if children.nil?
+	          children.each { |c| c[parent_prop] = p[parent_key] }
+	          all_children.push(*children)
+	          p.delete child_prop
+	        end
+	        all_children
+	      end
+			end
+		end
+	end
+end

data/lib/seiso/import_master/importers/node_importer.rb

@@ -0,0 +1,79 @@
+require_relative "base_importer"
+
+module Seiso
+  class ImportMaster
+    module Importers
+      
+      # Imports a node document into Seiso.
+      # 
+      # Each node doc corresponds to a service instance. This importer compares the nodes in Seiso with the
+      # nodes in the node doc, and removes anything from Seiso that isn't in the node doc. Then it imports
+      # the nodes from the node doc into Seiso.
+      #
+      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+      # License:: Apache 2.0
+      class NodeImporter < BaseImporter
+
+        def initialize(mapper, uri_factory_v1, rest_connector_v1)
+          @mapper = mapper
+          @uri_factory_v1 = uri_factory_v1
+          @rest_connector_v1 = rest_connector_v1
+        end
+
+        # Imports the node document.
+        def import(doc)
+          si_key = doc['serviceInstance']
+          nodes = doc['items']
+          puts "Importing node document for service instance #{si_key}"
+
+          delete_stale_nodes_from_seiso
+
+          # Node masters contain the service instance at the top-level, so enrich the nodes themselves
+          # with the service instance.
+          nodes.each do |n|
+            n['serviceInstance'] = si_key
+          end
+
+          nips = detach_children(nodes, 'node', 'name', 'ipAddresses')
+
+          seiso_nodes = nodes.map { |n| @mapper.seiso_node n }
+          nodes_uri = @uri_factory_v1.nodes_uri true
+          @rest_connector_v1.post(nodes_uri, seiso_nodes)
+
+#          import_nips_slowly_without_keepalive nips
+          import_nips_quickly_with_keepalive nips
+        end
+
+        private
+
+        def delete_stale_nodes_from_seiso
+          # TODO
+#          puts "TODO Delete stale nodes from seiso"
+        end
+
+        def import_nips_slowly_without_keepalive(nips)
+          seiso_nips = nips.map { |nip| @mapper.seiso_nip nip }
+          seiso_nips.each do |nip|
+            node_name = nip['node']['name']
+            ip_address = nip['ipAddress']
+            nip_uri = @uri_factory_v1.node_ip_address_uri(node_name, ip_address)
+            @rest_connector_v1.put(nip_uri, nip)
+          end
+        end
+
+        def import_nips_quickly_with_keepalive(nips)
+          seiso_nip_pairs = nips.map do |nip|
+            node_name = nip['node']
+            ip_address = nip['ipAddress']
+            {
+              "uri" => @uri_factory_v1.node_ip_address_uri(node_name, ip_address),
+              "resource" => @mapper.seiso_nip(nip)
+            }
+          end
+          @rest_connector_v1.put_all seiso_nip_pairs
+        end
+      end
+    end
+	end
+end

data/lib/seiso/import_master/importers/service_importer.rb

@@ -0,0 +1,135 @@
+require_relative "base_importer"
+
+module Seiso
+	class ImportMaster
+		module Importers
+
+			# Imports a service document into Seiso.
+			# 
+      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+      # License:: Apache 2.0
+			class ServiceImporter < BaseImporter
+
+				def initialize(
+            service_mapper,
+            basic_mapper,
+            uri_factory_v1,
+            uri_factory_v2,
+            rest_connector_v1,
+            rest_connector_v2)
+
+					@service_mapper = service_mapper
+          @basic_mapper = basic_mapper
+          @uri_factory_v1 = uri_factory_v1
+          @uri_factory_v2 = uri_factory_v2
+          @rest_connector_v1 = rest_connector_v1
+          @rest_connector_v2 = rest_connector_v2
+				end
+
+				# Imports the service document
+				def import(doc)
+					services = doc['items']
+					puts "Importing service document"
+
+					doc_links = detach_children(services, 'service', 'key', 'docLinks')
+
+					seiso_services = services.map { |s| @service_mapper.seiso_service s }
+          services_uri = @uri_factory_v1.services_uri true
+          @rest_connector_v1.post(services_uri, seiso_services)
+
+          import_doc_links doc_links
+        end
+
+        private
+
+        def import_doc_links(doc_links)
+          doc_links_by_service = group_doc_links_by_service doc_links
+
+          doc_links_by_service.each do |service_key, doc_links|
+            doc_links_uri = @uri_factory_v2.doc_links_uri service_key
+
+            # Get service's doc links so we can
+            # 1) Delete stale doc links
+            # 2) Avoid duplicate imports
+            seiso_doc_links_response = @rest_connector_v2.get(doc_links_uri)
+            seiso_doc_links = JSON.parse(seiso_doc_links_response.body)
+
+            to_delete = seiso_doc_links_to_delete(doc_links, seiso_doc_links)
+
+            # Delete stale doc links
+            to_delete.each do |seiso_doc_link|
+              href = seiso_doc_link['_links']['self']['href']
+              uri = URI.parse href
+              @rest_connector_v2.delete(uri)
+            end
+
+            # Classify remaining links as post or put
+            to_post = []
+            to_put = []
+            doc_links.each do |sdm_doc_link|
+              sdm_title = sdm_doc_link['title']
+              post_it = true
+              seiso_doc_links.each do |seiso_doc_link|
+                seiso_title = seiso_doc_link['title']
+                if sdm_title == seiso_title
+                  href = seiso_doc_link['_links']['self']['href']
+                  uri = URI.parse href
+                  new_seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
+                  to_put << {
+                    "uri" => uri,
+                    "resource" => new_seiso_doc_link
+                  }
+                  post_it = false
+                  break
+                end
+              end
+              to_post << sdm_doc_link if post_it
+            end
+
+            # Post new links
+            to_post.each do |sdm_doc_link|
+              seiso_doc_link = @basic_mapper.map_one('doc-links', sdm_doc_link)
+              @rest_connector_v2.post(doc_links_uri, seiso_doc_link)
+            end
+
+            # Put existing links
+            @rest_connector_v2.put_all(to_put)
+          end
+        end
+
+        def group_doc_links_by_service(doc_links)
+          doc_links_by_service = {}
+          doc_links.each do |dl|
+            service_key = dl['service']
+            service_doc_links = doc_links_by_service[service_key]
+            if service_doc_links.nil?
+              service_doc_links = []
+              doc_links_by_service[service_key] = service_doc_links
+            end
+            service_doc_links << dl
+          end
+          doc_links_by_service
+        end
+
+        def seiso_doc_links_to_delete(sdm_doc_links, seiso_doc_links)
+          to_delete = []
+          seiso_doc_links.each do |seiso_doc_link|
+            seiso_title = seiso_doc_link['title']
+            delete_it = true
+            sdm_doc_links.each do |sdm_doc_link|
+              sdm_title = sdm_doc_link['title']
+              if sdm_title == seiso_title
+                delete_it = false
+                break
+              end
+            end
+            to_delete << seiso_doc_link if delete_it
+          end
+          to_delete
+        end
+
+			end # class ServiceImporter
+		end # module Importers
+	end # class ImportMaster
+end # module Seiso