seiso-import_master 0.0.6 → 0.0.7

data/lib/seiso/import_master/util/uri_factory_v1.rb

@@ -0,0 +1,61 @@
+require "uri"
+
+module Seiso
+  class ImportMaster
+    module Util
+
+      # Seiso API v1 URI factory.
+      # 
+      # URI templating (embodied here) is a temporary strategy for obtaining URIs. The plan is
+      # to adopt something like a Traverson-style strategy.
+      # 
+      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+      # License:: Apache 2.0
+      class UriFactoryV1
+
+        def initialize(base_uri)
+          @base_uri = base_uri
+        end
+
+        def nodes_uri(batch)
+          if batch
+            uri "/nodes?mode=batch"
+          else
+            uri "/nodes"
+          end
+        end
+
+        def node_uri(name)
+          uri "/nodes/#{name}"
+        end
+        
+        def node_ip_address_uri(node_name, ip_address)
+          uri "/nodes/#{node_name}/ip-addresses/#{ip_address}"
+        end
+
+        def services_uri(batch)
+          if batch
+            uri "/services?mode=batch"
+          else
+            uri "/services"
+          end
+        end
+
+        def service_instances_uri(batch)
+          if batch
+            uri "/service-instances?mode=batch"
+          else
+            uri "/service-instances"
+          end
+        end
+
+        private
+
+        def uri(path)
+          URI.parse "#{@base_uri}#{path}"
+        end
+      end
+    end
+  end
+end

data/lib/seiso/import_master/util/uri_factory_v2.rb

@@ -0,0 +1,34 @@
+require "uri"
+
+module Seiso
+  class ImportMaster
+    module Util
+
+      # Seiso API v2 URI factory.
+      # 
+      # URI templating (embodied here) is a temporary strategy for obtaining URIs. The plan is
+      # to adopt something like a Traverson-style strategy.
+      # 
+      # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+      # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+      # License:: Apache 2.0
+      class UriFactoryV2
+
+        def initialize(base_uri)
+          @base_uri = base_uri
+        end
+
+        def doc_links_uri(service_key)
+          uri "/services/#{service_key}/doc-links"
+        end
+
+        private
+
+        def uri(path)
+          URI.parse "#{@base_uri}#{path}"
+        end
+        
+      end # class UriFactoryV2
+    end # module Util
+  end # class ImportMaster
+end # module Seiso

data/lib/seiso/import_master/validators/base_validator.rb

@@ -0,0 +1,19 @@
+require_relative "../errors/invalid_document_error"
+
+module Seiso
+	class ImportMaster
+		module Validators
+			
+		  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+		  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+		  # License:: Apache 2.0
+			class BaseValidator
+
+				# Raises an InvalidDocumentError with the given message.
+				def error(msg)
+          raise Seiso::ImportMaster::Errors::InvalidDocumentError.new msg
+				end
+			end
+		end
+	end
+end

data/lib/seiso/import_master/validators/node_validator.rb

@@ -0,0 +1,46 @@
+require_relative "base_validator"
+
+module Seiso
+	class ImportMaster
+		module Validators
+
+		  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+		  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+		  # License:: Apache 2.0
+			class NodeValidator < BaseValidator
+
+				def validate(doc)
+					validate_si_key doc
+					validate_nodes doc
+	  		end
+
+	  		private
+
+	  		def validate_si_key(doc)
+	        si_key = doc['serviceInstance']
+	        
+	        if si_key.nil?
+	          error "Node document must contain a serviceInstance field containing the service instance key."
+	        end
+	  		end
+
+	  		def validate_nodes(doc)
+	        nodes = doc['items']
+
+	        if nodes.nil?
+	          error "Node document must contain an items field containing the service instance's nodes."
+	        end
+
+	  			# Fail if the node has a serviceInstance field. This is legacy, and we don't want it anymore, because
+	  			# people will think that they can move a node from one service instance to another by changing this field.
+	  			nodes.each do |n|
+	  				if n['serviceInstance']
+	            error "Individual nodes must not contain a serviceInstance field. " \
+	                "Specify the service instance using the document-level serviceInstance field."
+	  				end
+	  			end
+				end
+			end
+		end
+	end
+end

data/lib/seiso/import_master/validators/service_validator.rb

@@ -0,0 +1,45 @@
+require_relative "base_validator"
+
+module Seiso
+	class ImportMaster
+		module Validators
+
+		  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+		  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+		  # License:: Apache 2.0
+			class ServiceValidator < BaseValidator
+
+				def validate(doc)
+					validate_services doc
+				end
+
+				private
+
+				def validate_services(doc)
+					services = doc['items']
+
+					if services.nil?
+						error "Service document must contain an items field containing the services."
+					end
+
+					services.each { |s| validate_service s }
+				end
+
+				def validate_service(s)
+					key = s['key']
+
+					if key.nil?
+						error "Service must have a key"
+					end
+
+					name = s['name']
+
+					if name.nil?
+						error "Service must have a name"
+					end
+				end
+			end
+		end
+	end
+end
+		

data/seiso-import_master.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "seiso-import_master"
-  spec.version       = "0.0.6"
+  spec.version       = "0.0.7"
   spec.authors       = ["Willie Wheeler"]
   spec.email         = ["wwheeler@expedia.com"]
   spec.summary       = "Imports Seiso data master files into Seiso."
@@ -17,8 +17,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '~> 1.9.3'
+
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 
   spec.add_runtime_dependency "seiso-connector", "~> 0.1.0"
+  spec.add_runtime_dependency "require_all", "~> 1.3.2"
 end

data/test/test_master_item_mapper.rb

@@ -1,14 +1,14 @@
 require 'minitest/autorun'
-require 'seiso/import_master/master_item_mapper'
+require 'seiso/import_master/mappers/master_item_mapper'
 
 # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
 # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
 # License:: Apache 2.0
-
 class TestMasterItemMapper < MiniTest::Unit::TestCase
 
+=begin
   def setup
-    @mapper = Seiso::ImportMaster::MasterItemMapper.new
+    @mapper = MasterItemMapper.new
   end
 
   def test_map_all_illegal_type
@@ -120,33 +120,6 @@ class TestMasterItemMapper < MiniTest::Unit::TestCase
     assert_equal(from['platformVersion'], to['platformVersion'])
   end
 
-  def test_map_node
-    from = {
-      'name' => 'seiso01-dev',
-      'serviceInstance' => 'seiso-dev',
-      'machine' => 'ip-1-2-3-4',
-      'ipAddresses' => [
-        {
-          'ipAddressRole' => 'internal',
-          'ipAddress' => '1.2.10.1'
-        }, {
-          'ipAddressRole' => 'partners',
-          'ipAddress' => '1.2.10.2'
-        }
-      ]
-    }
-    to = @mapper.map_one('nodes', from)
-    assert_equal(from['name'], to['name'])
-    refute_nil(to['serviceInstance'])
-    refute_nil(to['machine'])
-
-    # TODO Move these to a test_ip_address method
-#    assert_equal(from['ipAddresses'][0]['ipAddressRole'], to['ipAddresses'][0]['ipAddressRole']['name'])
-#    assert_equal(from['ipAddresses'][0]['ipAddress'], to['ipAddresses'][0]['ipAddress'])
-#    assert_equal(from['ipAddresses'][1]['ipAddressRole'], to['ipAddresses'][1]['ipAddressRole']['name'])
-#    assert_equal(from['ipAddresses'][1]['ipAddress'], to['ipAddresses'][1]['ipAddress'])
-  end
-
   def test_map_region
     from = {
       'key' => 'amazon-us-east-1',
@@ -279,4 +252,6 @@ class TestMasterItemMapper < MiniTest::Unit::TestCase
     assert_equal(from['key'], to['key'])
     assert_equal(from['name'], to['name'])
   end
+=end
+
 end

data/test/test_node_mapper.rb

@@ -0,0 +1,40 @@
+require "minitest/autorun"
+require "seiso/import_master/mappers/node_mapper"
+
+# Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
+# Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+# License:: Apache 2.0
+class TestNodeMapper < MiniTest::Unit::TestCase
+
+	def setup
+		@mapper = Seiso::ImportMaster::Mappers::NodeMapper.new
+	end
+
+	def test_map_node
+    from = {
+      'name' => 'seiso01-dev',
+      'machine' => 'ip-1-2-3-4',
+      'ipAddresses' => [
+        {
+          'ipAddressRole' => 'internal',
+          'ipAddress' => '1.2.10.1'
+        }, {
+          'ipAddressRole' => 'partners',
+          'ipAddress' => '1.2.10.2'
+        }
+      ]
+    }
+    to = @mapper.seiso_node(from)
+    assert_equal(from['name'], to['name'])
+    refute_nil(to['machine'])
+
+    # TODO Move these to a test_ip_address method
+#    assert_equal(from['ipAddresses'][0]['ipAddressRole'], to['ipAddresses'][0]['ipAddressRole']['name'])
+#    assert_equal(from['ipAddresses'][0]['ipAddress'], to['ipAddresses'][0]['ipAddress'])
+#    assert_equal(from['ipAddresses'][1]['ipAddressRole'], to['ipAddresses'][1]['ipAddressRole']['name'])
+#    assert_equal(from['ipAddresses'][1]['ipAddress'], to['ipAddresses'][1]['ipAddress'])
+	end
+
+	def test_map_node_ip_address
+	end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seiso-import_master
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Willie Wheeler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.2
 description: Supports self-service data management via version control, e.g. by using
   GitHub.
 email:
@@ -69,11 +83,26 @@ files:
 - Rakefile
 - bin/seiso-import-master
 - lib/seiso/import_master.rb
-- lib/seiso/import_master/master_item_mapper.rb
-- refresh
+- lib/seiso/import_master/errors/invalid_document_error.rb
+- lib/seiso/import_master/importers/base_importer.rb
+- lib/seiso/import_master/importers/node_importer.rb
+- lib/seiso/import_master/importers/service_importer.rb
+- lib/seiso/import_master/importers/service_instance_importer.rb
+- lib/seiso/import_master/importers/simple_importer.rb
+- lib/seiso/import_master/mappers/master_item_mapper.rb
+- lib/seiso/import_master/mappers/node_mapper.rb
+- lib/seiso/import_master/mappers/service_instance_mapper.rb
+- lib/seiso/import_master/mappers/service_mapper.rb
+- lib/seiso/import_master/util/rest_connector.rb
+- lib/seiso/import_master/util/uri_factory_v1.rb
+- lib/seiso/import_master/util/uri_factory_v2.rb
+- lib/seiso/import_master/validators/base_validator.rb
+- lib/seiso/import_master/validators/node_validator.rb
+- lib/seiso/import_master/validators/service_validator.rb
 - sample_conf/seiso.yml.sample
 - seiso-import_master.gemspec
 - test/test_master_item_mapper.rb
+- test/test_node_mapper.rb
 homepage: http://seiso.io
 licenses:
 - Apache-2.0
@@ -84,9 +113,9 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ~>
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
@@ -100,3 +129,4 @@ specification_version: 4
 summary: Imports Seiso data master files into Seiso.
 test_files:
 - test/test_master_item_mapper.rb
+- test/test_node_mapper.rb

data/lib/seiso/import_master/master_item_mapper.rb

@@ -1,295 +0,0 @@
-module Seiso
-  class ImportMaster
-
-    # Maps the data master format to the Seiso API format.
-    #
-    # Author:: Willie Wheeler
-    # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-    # License:: Apache 2.0
-    class MasterItemMapper
-    
-      def initialize
-        @mappers = {
-          'data-centers' => data_center_mapper,
-          'doc-links' => doc_link_mapper,
-          'environments' => environment_mapper,
-          'health-statuses' => health_status_mapper,
-          'infrastructure-providers' => infrastructure_provider_mapper,
-          'ip-address-roles' => ip_address_role_mapper,
-          'load-balancers' => load_balancer_mapper,
-          'machines' => machine_mapper,
-          'nodes' => node_mapper,
-          'node-ip-addresses' => node_ip_address_mapper,
-          'regions' => region_mapper,
-          'rotation-statuses' => rotation_status_mapper,
-          'services' => service_mapper,
-          'service-groups' => service_group_mapper,
-          'service-instances' => service_instance_mapper,
-          'service-instance-ports' => service_instance_port_mapper,
-          'service-types' => service_type_mapper,
-          'status-types' => status_type_mapper
-        }
-      end
-
-      # Maps a list of items.
-      # - type: Item type
-      # - items: Item list
-      def map_all(type, items)
-        mapper = mapper_for type
-        result = []
-        items.each { |i| result << mapper.call(i) }
-        result
-      end
-      
-      # Maps a single item.
-      # - type: Item type
-      # - item: Item
-      def map_one(type, item)
-        mapper_for(type).call item
-      end
-      
-      private
-      
-      # Returns a mapper lambda for the specified type.
-      def mapper_for(type)
-        mapper = @mappers[type]
-        raise ArgumentError, "Unknown type: #{type}" if mapper.nil?
-        mapper
-      end
-      
-      def data_center_mapper
-        ->(dc) {
-          {
-            'key' => dc['key'],
-            'name' => dc['name'],
-            'region' => { 'key' => dc['region'] }
-          }
-        }
-      end
-      
-      def doc_link_mapper
-        ->(dl) {
-          {
-            'title' => dl['title'],
-            'href' => dl['href'],
-            'description' => dl['description']
-          }
-        }
-      end
-
-      def environment_mapper
-        ->(e) {
-          {
-            'key' => e['key'],
-            'name' => e['name'],
-            'aka' => e['aka'],
-            'description' => e['description'],
-            
-            # FIXME Deprecated, don't use.
-            'rank' => e['rank']
-          }
-        }
-      end
-      
-      def health_status_mapper
-        ->(hs) {
-          {
-            'key' => hs['key'],
-            'name' => hs['name'],
-            'statusType' => { 'key' => hs['statusType'] }
-          }
-        }
-      end
-      
-      def infrastructure_provider_mapper
-        ->(ip) {
-          {
-            'key' => ip['key'],
-            'name' => ip['name']
-          }
-        }
-      end
-      
-      def ip_address_role_mapper
-        # Suppressing IP addresses since we don't import those from master files.
-        ->(r) {
-          {
-            'serviceInstance' => { 'key' => r['serviceInstance'] },
-            'name' => r['name'],
-            'description' => r['description']
-          }
-        }
-      end
-      
-      def load_balancer_mapper
-        ->(lb) {
-          seiso_lb = {
-            'name' => lb['name'],
-            'type' => lb['type'],
-            'ipAddress' => lb['ipAddress'],
-            'apiUrl' => lb['apiUrl']
-          }
-          
-          dc = lb['dataCenter']
-          seiso_lb['dataCenter'] = { 'key' => dc } unless dc.nil?
-          
-          seiso_lb
-        }
-      end
-      
-      def machine_mapper
-        ->(m) {
-          {
-            'name' => m['name'],
-            'ipAddress' => m['ipAddress'],
-            'fqdn' => m['fqdn'],
-            'hostname' => m['hostname'],
-            'domain' => m['domain'],
-            'os' => m['os'],
-            'platform' => m['platform'],
-            'platformVersion' => m['platformVersion']
-          }
-        }
-      end
-      
-      def node_mapper
-        ->(n) {
-          seiso_node = {
-            'name' => n['name'],
-            'serviceInstance' => { 'key' => n['serviceInstance'] }
-          }
-
-          machine = n['machine']
-          seiso_node['machine'] = { 'name' => machine } unless machine.nil?
-
-          seiso_node
-        }
-      end
-      
-      def node_ip_address_mapper
-        # Currently suppressing rotation status and endpoints since we don't import those from master files
-        ->(nip) {
-          {
-            'node' => { 'name' => nip['node'] },
-            'ipAddressRole' => { 'name' => nip['ipAddressRole'] },
-            'ipAddress' => nip['ipAddress']
-          }
-        }
-      end
-      
-      def region_mapper
-        ->(r) {
-          {
-            'key' => r['key'],
-            'name' => r['name'],
-            'regionKey' => r['regionKey'],
-            'infrastructureProvider' => { 'key' => r['infrastructureProvider'] }
-          }
-        }
-      end
-      
-      def rotation_status_mapper
-        ->(rs) {
-          {
-            'key' => rs['key'],
-            'name' => rs['name'],
-            'statusType' => { 'key' => rs['statusType'] }
-          }
-        }
-      end
-      
-      def service_mapper
-        ->(s) {
-          seiso_service = {
-            'key' => s['key'],
-            'name' => s['name'],
-            'description' => s['description'],
-            'platform' => s['platform'],
-            'scmRepository' => s['scmRepository']
-          }
-
-          group = s['group']
-          seiso_service['group'] = { 'key' => group } unless group.nil?
-
-          type = s['type']
-          seiso_service['type'] = { 'key' => type } unless type.nil?
-
-          owner = s['owner']
-          seiso_service['owner'] = { 'username' => owner } unless owner.nil?
-
-          seiso_service
-        }
-      end
-      
-      def service_group_mapper
-        ->(sg) {
-          {
-            'key' => sg['key'],
-            'name' => sg['name']
-          }
-        }
-      end
-      
-      def service_instance_mapper
-        ->(si) {
-          key = si['key']
-          ip_address_roles = si['ipAddressRoles']
-          ports = si['ports']
-          
-          seiso_si = {
-            'key' => key,
-            'service' => { 'key' => si['service'] },
-            'environment' => { 'key' => si['environment'] },
-            'loadBalanced' => si['loadBalanced'],
-            'minCapacityDeploy' => si['minCapacityDeploy'],
-            'minCapacityOps' => si['minCapacityOps'],
-            
-            # FIXME Deprecated. This is a hardcoded field for an internal Expedia app, and we will remove it.
-            'eosManaged' => (si['eosManaged'] || false),
-            
-            # FIXME Deprecated. This is the old name for the minCapacityOps field.
-            'requiredCapacity' => si['minCapacityOps']
-          }
-
-          data_center = si['dataCenter']
-          seiso_si['dataCenter'] = { 'key' => data_center } unless data_center.nil?
-          
-          load_balancer = si['loadBalancer']
-          seiso_si['loadBalancer'] = { 'name' => load_balancer } unless load_balancer.nil?
-
-          seiso_si
-        }
-      end
-      
-      def service_instance_port_mapper
-        # Suppressing endpoints since we don't import those from master files.
-        ->(p) {
-          {
-            'serviceInstance' => { 'key' => p['serviceInstance'] },
-            'number' => p['number'],
-            'protocol' => p['protocol'],
-            'description' => p['description']
-          }
-        }
-      end
-      
-      def service_type_mapper
-        ->(st) {
-          {
-            'key' => st['key'],
-            'name' => st['name']
-          }
-        }
-      end
-      
-      def status_type_mapper
-        ->(st) {
-          {
-            'key' => st['key'],
-            'name' => st['name']
-          }
-        }
-      end
-    end
-  end
-end