seiso-import_master 0.0.12 → 3.0.0.M1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Y2M4MmJmOWJiYTM0NDliNTk3MDZjODJmODcyZjcwNTcxOWQ2ZjAzNA==
+    MTk2ZjJlYzk0YTA2YzVhN2IwOWEyZWM3NWM3Y2MwMjc3MjI0ZTNkOA==
   data.tar.gz: !binary |-
-    NjRkZDA2NmUyOGY1MjI4OWE1MzljNjdjYjNkZmMwOTNiMmFkYTFjNQ==
+    ZTM0NDk1MTQ1YmIxM2Y5OTdmMzc4ZTE4MWY5ZmU0NjM4MjcxYmY3Ng==
 SHA512:
   metadata.gz: !binary |-
-    NmMyOTUyZWIxNDM1NDE0M2IyNDFjMzgzOWZkYTc4NGRkZDMyNzQ4ZjBjYzFl
-    YTQwMDA1MmVmMjFhNjFmM2EzOTljZjkxMjk4YjNhNDhkMzM3YWE0YTlmNTQ5
-    OWU0ZGFmNzU5NWVmZTI5Mjc4ZWYzZmVhMzU0MmY0NGY4ODM5YTA=
+    YTkwNmE2N2E5M2FlMzZkN2FmNGZjOGEyODFjNmNmOTJmMWMxMThiYmJmMTY2
+    ZGY2YmQwZTIxMjU3MTkwYWJjMGFlYjU0YWI3ZjM3MjRlYmVlYTUyMGVkY2Ux
+    MDQ0NDUzODhlZmVjYjFkNDJhNGQ5Mjk3N2Q5YjRmNTNjMTc5MDM=
   data.tar.gz: !binary |-
-    NjE1NTIyMTNkZjA1MmQ0NzgzN2JhMTg5YzE3YjI5MmM0NTIzNjg2N2E4YjY5
-    M2JkMjVhOGM2ZTJiN2U5ZjQ1NzdlN2JiMGJmMDczODNiZGM5ODhlMjliMDUy
-    YmQ4NzY4YjRhYTZhODE4OTVjZDYxYjdjODYzZWIzNzBmOTVkMWM=
+    MTI5ZGI3Y2E4NmFkMzg4YjlkMGRjNWQwMDcyODMxMWQxZGU0ZTIxNjU2OWQw
+    OGE5YjM4YTEwNzhlNzM1NjhjMDcyMTdjNGNjNzM3YzA4MDMzNTcxMmM4M2Yx
+    MmZmMjFmOTk4MDRlMDFlZjllN2I3NTRhOThjMDk3YWNmYzBjNTc=

data/.gitignore

@@ -14,3 +14,4 @@
 *.a
 mkmf.log
 /.project
+/.idea

data/.travis.yml

@@ -1,3 +1,6 @@
 language: ruby
 rvm:
   - 1.9.3
+
+before_install:
+  - gem install bundler

data/README.md

@@ -10,26 +10,20 @@ See [Manage Your Service Data with GitHub, Jenkins & Seiso](http://seiso.io/guid
 
 See [Seiso Data Master Schemas](http://seiso.io/docs/current/user-manual/sdm-schemas/) for the data master schemas.
 
-## Installation
-
-Add this line to your application's Gemfile:
+## Installation preliminaries
 
-```ruby
-gem 'seiso-import_master'
-```
+First, I recommend using [RVM](http://rvm.io/) to install/manage your rubies so you can run multiple rubies without conflict.
 
-And then execute:
+You will need Ruby 1.9.3 as well as [Bundler](http://bundler.io).
 
-    $ bundle
-
-Or install it yourself as:
+## Installation
 
     $ gem install seiso-import_master
 
 ## Usage
 
 1. Create a directory `~/.seiso-importers`
-2. Place appropriately modified copy of `seiso.yml.sample` in there.
+2. Place appropriately modified copy of `seiso3.yml.sample` in there. Note that you will need to get Seiso credentials from your Seiso administrator.
 3. Run `seiso-import-master file [, file2, ...]` to perform the import. Note that you can use `-f yaml` for YAML files (the default is `-f json`).
 
 ## Contributing
@@ -53,3 +47,10 @@ In particular, run the unit tests using
 Build and install the gem using Rake:
 
     $ rake install
+
+For local snapshot installations, you may want to clear out existing versions of the gem first.
+
+## TODO
+
+Figure out whether we want to use dependency injection for this project. There's debate within the Ruby community
+whether DI makes sense for Ruby.

data/Rakefile

@@ -1,5 +1,5 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 task :default => [ :test ]
 
@@ -11,7 +11,7 @@ end
 
 Rake::TestTask.new do |t|
   t.libs << 'test'
-  t.test_files = FileList[ 'test/test_*.rb' ]
+  t.test_files = FileList[ 'test/**/test_*.rb' ]
   t.verbose = true
   t.warning = true
 end

data/bin/seiso-import-master

@@ -3,31 +3,33 @@
 # vi: set ft=ruby :
 
 require 'optparse'
-require 'seiso/import_master'
 require 'yaml'
+require 'seiso/import_master'
 
 # Imports Seiso data master files into Seiso.
 #
 # Author:: Willie Wheeler
-# Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
+# Copyright:: Copyright (c) 2014-2016 Expedia, Inc.
 # License:: Apache 2.0
 
+log = Seiso::ImportMaster::Util::Logger.new "seiso-import-master"
+
 options = {}
 option_parser = OptionParser.new do |opts|
   executable_name = File.basename $PROGRAM_NAME
   opts.banner = "Import one or more Seiso data master files into Seiso.
 
 Usage: #{executable_name} [options] master_file ..."
-  options[:seiso_settings_path] = "#{Dir.home}/.seiso-importers/seiso.yml"
-  opts.on("--seiso SEISO_SETTINGS_PATH", "Path to Seiso API settings file") do |path|
-    options[:seiso_settings_path] = path
+  options[:seiso3_settings_path] = "#{Dir.home}/.seiso-importers/seiso3.yml"
+  opts.on("--seiso3 SEISO3_SETTINGS_PATH", "Path to Seiso API v3 settings file") do |path|
+    options[:seiso3_settings_path] = path
   end
 
   options[:format] = 'json'
   opts.on("-f FORMAT", "--format FORMAT", "Document format, either 'json' (default) or 'yaml'") do |format|
     options[:format] = format
   end
-  
+
   opts.on("-h", "--help", "Display this screen") do
     puts opts
     exit
@@ -37,15 +39,30 @@ end
 begin
   option_parser.parse!
 rescue OptionParser::InvalidOption => e
-  abort "Invalid option.\n\n#{option_parser.help}"
+  log.error "Invalid option."
+  puts option_parser.help
+  abort
 end
 
-puts "Using: #{options[:seiso_settings_path]}"
-puts "Using: format=#{options[:format]}"
+log.info "Using: #{options[:seiso3_settings_path]}"
+log.info "Using: format=#{options[:format]}"
 
 files = ARGV
-abort "Error: You must supply a master file.\n\n#{option_parser.help}" if files.empty?
+if files.empty?
+  log.error "You must supply a master file."
+  puts option_parser.help
+  abort
+end
+
+seiso3_settings = YAML.load_file(options[:seiso3_settings_path])
+importer = Seiso::ImportMaster.new seiso3_settings
+result = importer.import_files(files, options[:format])
 
-settings = YAML.load_file(options[:seiso_settings_path])
-importer = Seiso::ImportMaster.new settings
-importer.import_files(files, options[:format])
+if result == "success"
+  log.success "Import succeeded."
+elsif result == "partial"
+  log.warn "Import succeeded, but with issues."
+else
+  log.error "Import failed."
+  abort
+end

data/lib/seiso/import_master.rb

@@ -1,114 +1,138 @@
-require "json"
-require "require_all"
-require "yaml"
-require "seiso/connector"
-
-require_rel "import_master"
+require 'json'
+require 'yaml'
+require 'hyper_resource'
+require 'require_all'
+require_rel './import_master'
 
+# Imports Seiso data master files into Seiso.
+#
+# Author:: Willie Wheeler
+# Copyright:: Copyright (c) 2014-2016 Expedia, Inc.
+# License:: Apache 2.0
 module Seiso
-
-  # Imports Seiso data master files into Seiso.
-  #
-  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-  # License:: Apache 2.0
   class ImportMaster
-    
-    # Initializes the importer with a Seiso connector.
-    def initialize(seiso_settings)
+
+    def initialize(seiso3_settings)
+      @log = Util::Logger.new "ImportMaster"
       @loaders = {
         'json' => ->(file) { JSON.parse(IO.read(file)) },
         'yaml' => ->(file) { YAML.load_file file }
       }
+      @v3_api = HyperResource.new(
+        root: seiso3_settings['base_uri'],
+        headers: {
+          'Accept' => 'application/hal+json',
+          'Content-Type' => 'application/hal+json'
+        },
+        auth: {
+          basic: [ seiso3_settings['username'], seiso3_settings['password'] ]
+        }
+      )
+      @resolver = Util::ItemResolver.new @v3_api
+      @rest_util = Util::RestUtil.new
 
-      # TODO Add other validators
-      @validators = {
-        'nodes' => Validators::NodeValidator.new,
-        'services' => Validators::ServiceValidator.new
-      }
-
-      basic_mapper = Mappers::MasterItemMapper.new
-      node_mapper = Mappers::NodeMapper.new
-      service_mapper = Mappers::ServiceMapper.new
-      service_instance_mapper = Mappers::ServiceInstanceMapper.new
-
-      seiso = Seiso::Connector.new seiso_settings
-      uri_factory_v1 = uri_factory_v1 seiso_settings
-      uri_factory_v2 = uri_factory_v2 seiso_settings
-      rest_connector_v1 = rest_connector(seiso_settings, "application/json")
-      rest_connector_v2 = rest_connector(seiso_settings, "application/hal+json")
-
-      @simple_importer = Importers::SimpleImporter.new(basic_mapper, seiso, uri_factory_v1, rest_connector_v1)
-
-      # TODO Clean this up. Just use v2 URI factory and connector.
-      @importers = {
-        'nodes' => Importers::NodeImporter.new(node_mapper, uri_factory_v1, rest_connector_v1),
-        'services' => Importers::ServiceImporter.new(
-            basic_mapper,
-            service_mapper,
-            uri_factory_v1,
-            uri_factory_v2,
-            rest_connector_v1,
-            rest_connector_v2),
-        'service-instances' => Importers::ServiceInstanceImporter.new(
-            service_instance_mapper,
-            uri_factory_v1,
-            uri_factory_v2,
-            seiso,
-            rest_connector_v1,
-            rest_connector_v2)
-      }
+      # Importer cache, since they're expensive to initialize
+      @importers = {}
     end
 
     # Imports a list of master files in order. Legal formats are 'json' (default) and 'yaml'.
+    # Returns true if all file imports succeeded and false otherwise.
     def import_files(files, format = 'json')
-      loop do
-        file = files.pop
-        puts "Processing: #{file}"
-        import_file(file, format)
-        break if files.empty?
+      begin
+        all_failed = true
+        all_success = true
+        loop do
+          file = files.pop
+          success = import_file(file, format)
+          if success
+            all_failed = false
+          else
+            all_success = false
+          end
+          break if files.empty?
+        end
+        if all_success
+          return "success"
+        elsif all_failed
+          return "failed"
+        else
+          return "partial"
+        end
+      rescue Interrupt => e
+        @log.error "Import interrupted."
+        return "failed"
       end
     end
-    
+
+    private
+
     # Imports a data master file. Legal formats are 'json' (default) and 'yaml'.
+    # Returns true if the import succeeded and false otherwise.
     def import_file(file, format = 'json')
-      loader = @loaders[format]
-      raise ArgumentError, "Illegal format: #{format}" if loader.nil?
-      doc = loader.call(file)
-      type = doc['type']
-      validator = @validators[type]
-      validator.validate(doc) unless validator.nil?
-      (@importers[type] || @simple_importer).import doc
-    end
+      @log.info "Importing file #{file}"
+      begin
+        # Include this in the begin/rescue as it can raise a JSON::ParserError
+        loader = @loaders[format]
 
-    private
+        fail ArgumentError, "Illegal format: #{format}" if loader.nil?
 
-    def uri_factory_v1(settings)
-      host = settings['host']
-      port = settings['port']
-      use_ssl = settings['use_ssl'] || false
-      scheme = use_ssl ? "https" : "http"
-      base_uri = "#{scheme}://#{host}:#{port}/v1"
-      Seiso::Connector::UriFactoryV1.new base_uri
-    end
+        doc = loader.call file
+        type = doc['type']
+        get_importer(type).import doc
+        return true
 
-    def uri_factory_v2(settings)
-      host = settings['host']
-      port = settings['port']
-      use_ssl = settings['use_ssl'] || false
-      scheme = use_ssl ? "https" : "http"
-      base_uri = "#{scheme}://#{host}:#{port}/v2"
-      Seiso::Connector::UriFactoryV2.new base_uri
+      # Rescue StandardError here, not Exception, since we want ctrl-c to stop the program.
+      rescue StandardError => e
+        @log.error "Failed to import file #{file}: #{e.message}"
+
+        # FIXME For now, just re-raise the exception, as we want to see where the bugs are.
+        return false
+#        raise e
+      end
     end
 
-    def rest_connector(settings, media_type)
-      host = settings['host']
-      port = settings['port']
-      use_ssl = settings['use_ssl'] || false
-      ignore_cert = settings['ignore_cert'] || false
-      username = settings['username']
-      password = settings['password']
-      Seiso::Connector::RestConnector.new(host, port, use_ssl, ignore_cert, username, password, media_type)
+    def get_importer(type)
+      # TODO Do a cleaner impl here--one that doesn't violate the open-closed principle. Note that we *don't* want to
+      # create these eagerly, as each one involves an expensive initialization.
+      importer = @importers[type]
+
+      if importer.nil?
+        importer = case type
+        when 'data-centers'
+          Importers::DataCenterImporter.new(@v3_api, @rest_util, @resolver)
+        when 'environments'
+          Importers::EnvironmentImporter.new(@v3_api, @rest_util, @resolver)
+        when 'health-statuses'
+          Importers::HealthStatusImporter.new(@v3_api, @rest_util, @resolver)
+        when 'infrastructure-providers'
+          Importers::InfrastructureProviderImporter.new(@v3_api, @rest_util, @resolver)
+        when 'load-balancers'
+          Importers::LoadBalancerImporter.new(@v3_api, @rest_util, @resolver)
+        when 'machines'
+          Importers::MachineImporter.new(@v3_api, @rest_util, @resolver)
+        when 'nodes'
+          Importers::NodeImporter.new(@v3_api, @rest_util, @resolver)
+        when 'regions'
+          Importers::RegionImporter.new(@v3_api, @rest_util, @resolver)
+        when 'rotation-statuses'
+          Importers::RotationStatusImporter.new(@v3_api, @rest_util, @resolver)
+        when 'services'
+          Importers::ServiceImporter.new(@v3_api, @rest_util, @resolver)
+        when 'service-groups'
+          Importers::ServiceGroupImporter.new(@v3_api, @rest_util, @resolver)
+        when 'service-instances'
+          Importers::ServiceInstanceImporter.new(@v3_api, @rest_util, @resolver)
+        when 'service-types'
+          Importers::ServiceTypeImporter.new(@v3_api, @rest_util, @resolver)
+        when 'status-types'
+          Importers::StatusTypeImporter.new(@v3_api, @rest_util, @resolver)
+        else
+          raise ArgumentError, "Illegal type: #{type}"
+        end
+        @importers['type'] = importer
+      end
+
+      importer
     end
   end
 end

data/lib/seiso/import_master/importers.rb

@@ -0,0 +1,5 @@
+module Seiso
+  class ImportMaster
+    module Importers; end
+  end
+end

data/lib/seiso/import_master/importers/base_importer.rb

@@ -1,31 +1,92 @@
-module Seiso
-	class ImportMaster
-		module Importers
-
-		  # Author:: Willie Wheeler (mailto:wwheeler@expedia.com)
-		  # Copyright:: Copyright (c) 2014-2015 Expedia, Inc.
-		  # License:: Apache 2.0
-			class BaseImporter
-
-	      # Enriches children with a link to the parent, detaches them from the parent, and returns
-	      # all detached children.
-        # 
-        # - parents: List of items whose children we will detach.
-        # - parent_prop: Name of the parent property on the children.
-        # - parent_key: Name that Seiso expects for the parent key.
-        # - child_prop: Name of the child property on the items.
-	      def detach_children(parents, parent_prop, parent_key, child_prop)
-	        all_children = []
-	        parents.each do |p|
-	          children = p[child_prop]
-	          next if children.nil?
-	          children.each { |c| c[parent_prop] = p[parent_key] }
-	          all_children.push(*children)
-	          p.delete child_prop
-	        end
-	        all_children
-	      end
-			end
-		end
-	end
+require 'hyper_resource'
+
+class Seiso::ImportMaster
+
+  # Base importer for type-specific importers.
+  #
+  # Subclasses must set the @repo_resource variable to point to the type's repository resource.
+  #
+  # Additionally, subclasses must provide the following type-specific callback implementations:
+  #
+  # - to_search_params(doc_item, context) : Returns a hash containing HTTP params for the resource search.
+  # - find_resource(search_params) : Takes the search params above and returns the corresponding resource.
+  #
+  # If the type in question has child resources (e.g., service instances have IP address roles and service instance
+  # ports), then subclasses should provide the following callback implementations to handle them:
+  #
+  # - context_for(parent_resource, parent_context) : Returns a new context to support importing child resources.
+  # - import_children_of(doc_item, context) : Imports the child resources.
+  #
+  # See service_instance_importer.rb for an example that includes all four callbacks.
+  #
+  # Author:: Willie Wheeler
+  # Copyright:: Copyright (c) 2014-2016 Expedia, Inc.
+  # License:: Apache 2.0
+  class Importers::BaseImporter
+
+    # Imports a list of document items into Seiso.
+    #
+    # - items : Items in document format
+    # - context : Import context
+    def import_items(doc_items, context)
+      # TODO Perform orphan deletion here, centrally, and recursively?
+      doc_items.each { |doc_item| import_item(doc_item, context) }
+    end
+
+    # Imports a single document item into Seiso. Returns the imported item's URI
+    #
+    # - item : Item in document format
+    # - context : Import context
+    def import_item(doc_item, context)
+      data = mapper.map(doc_item, context)
+      search_params = to_search_params(doc_item, context)
+      begin
+        resource = find_resource search_params
+
+        # In the Seiso API, PUT will leave any unspecified associations alone. So for example we can handle node health
+        # status and rotation status like this. When we have node build version, though, we will have to do something
+        # else, like turn it into an association or else PATCH here. [WLW]
+        rest_util.put(resource, data, search_params)
+
+      rescue HyperResource::ClientError => e
+        status = e.response.status
+        body = e.response.body
+        fail "Client error: status=#{status}, body=#{body}" unless status == 404
+
+        # Seiso API returns the created resource in the response body, so grab that.
+        # TODO Clean up this implementation. E.g. don't used nested exception handling, and don't create the logger
+        # here. [WLW]
+        begin
+          resource = rest_util.post(repo_resource, data, search_params)
+        rescue HyperResource::ClientError => e2
+          # HTTP 409 is typical. It happens when client references nonexistent entities.
+          status2 = e2.response.status
+          body2 = e2.response.body
+          log = Util::Logger.new "BaseImporter"
+          log.warn "Client error: status=#{status2}, body=#{body2}. Failed to import item."
+          return
+        end
+      end
+
+      # ================================================================================================================
+      # FIXME For puts, this is putting the *old* resource on the context, which isn't what we want! [WLW]
+      # ================================================================================================================
+
+      if respond_to?(:import_children_of)
+        new_context = respond_to?(:context_for) ? context_for(resource, context) : {}
+        # FIXME For NodeIpAddress, this is returning a resource whose self link is /search/findByNodeNameAndIpAddress,
+        # and we can't PUT to that.
+        import_children_of(doc_item, new_context)
+      end
+
+      resource.href
+    end
+
+    protected
+
+    attr_accessor :api
+    attr_accessor :rest_util
+    attr_accessor :mapper
+    attr_accessor :repo_resource
+  end
 end