hammer_cli_import 0.10.21

data/lib/hammer_cli_import/importtools.rb

@@ -0,0 +1,351 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Modules to help with imports. To be used as Extend/Import on classes that inherit
+# from module HammerCLIImport::BaseCommand.
+
+require 'logger'
+
+module ImportTools
+  module Repository
+    module Extend
+      def add_repo_options
+        option ['--synchronize'], :flag, 'Synchronize imported repositories', :default => false
+        option ['--wait'], :flag, 'Wait for repository synchronization to finish', :default => false
+
+        add_async_tasks_reactor_options
+
+        validate_options do
+          option(:option_synchronize).required if option(:option_wait).exist?
+        end
+      end
+    end
+
+    module Include
+      def repo_synced?(repo)
+        raise ArgumentError, 'nil is not a valid repository' if repo.nil?
+
+        info = lookup_entity(:repositories, repo['id'], true)
+        last_sync_result = info['last_sync']['result'] if info['last_sync']
+        return false unless (info['sync_state'] == 'finished' || last_sync_result == 'success')
+
+        ## (Temporary) workaround for 1131954
+        ## updated_at is updated after sync for some reason...
+        # begin
+        #   Time.parse(info['last_sync']) > Time.parse(info['updated_at'])
+        # rescue
+        #   false
+        # end
+        true
+      end
+
+      def sync_repo(repo)
+        return unless option_synchronize?
+        task = api_call(:repositories, :sync, {:id => repo['id']})
+        debug "Sync of repo #{repo['id']} started!"
+        return unless option_wait?
+        wait_for_task task['id']
+      end
+
+      def sync_repo2(repo)
+        task = api_call(:repositories, :sync, {:id => repo['id']})
+        debug "Sync of repo #{repo['id']} started!"
+        task['id']
+      rescue
+        uuid = workaround_1116063 repo['id']
+        info 'Sync already running!'
+        uuid
+      end
+
+      def with_synced_repo(repo, &block)
+        # So we can not give empty block
+        if block_given?
+          action = block
+        else
+          action = proc {}
+        end
+
+        # Already synchronized?  Do the Thing
+        # ElsIf asked to sync, sync-and-then-wait to Do the Thing
+        # Otherwise, shrug and Skip The Thing
+        if repo_synced?(repo)
+          action.call
+        elsif option_synchronize?
+          uuid = sync_repo2 repo
+          postpone_till([uuid], &action) if option_wait?
+        end
+      end
+
+      private
+
+      # When BZ 1116063 get fixed, this might
+      # be simplified using either
+      # > api_call(:repositories, :show, {'id' => 1})
+      # or maybe with
+      # > api_call(:sync, :index, {:repository_id => 1})
+      def workaround_1116063(repo_id)
+        res = api_call :foreman_tasks, :bulk_search, \
+                       :searches => [{:type => :resource, :resource_type => 'Katello::Repository', :resource_id => repo_id}]
+
+        res.first['results'] \
+          .select { |x| x['result'] != 'error' } \
+          .max_by { |x| Time.parse(x['started_at']) }['id']
+      end
+    end
+  end
+
+  module LifecycleEnvironment
+    module Include
+      def get_env(org_id, name = 'Library')
+        @lc_environments ||= {}
+        @lc_environments[org_id] ||= {}
+        unless @lc_environments[org_id][name]
+          res = api_call :lifecycle_environments, :index, {:organization_id => org_id, :name => name}
+          @lc_environments[org_id][name] = res['results'].find { |x| x['name'] == name }
+        end
+        @lc_environments[org_id][name]
+      end
+    end
+  end
+
+  module Task
+    module Include
+      # [uuid] -> {uuid => {:finished => bool, :progress => Float}}
+      def annotate_tasks(uuids)
+        ret = {}
+        get_tasks_statuses(uuids).each do |uuid, stat|
+          ret[uuid] = { :finished => stat['state'] == 'stopped',
+                        :progress => stat['progress']}
+        end
+        ret
+      end
+
+      private
+
+      # [uuid] -> {uuid: task_status}
+      def get_tasks_statuses(uuids)
+        searches = uuids.collect { |uuid| {:type => :task, :task_id => uuid} }
+        ret = api_call :foreman_tasks, :bulk_search, {:searches => searches}
+        statuses = {}
+        ret.each do |status_result|
+          status_result['results'].each do |task_info|
+            statuses[task_info['id']] = task_info
+          end
+        end
+        statuses
+      end
+    end
+  end
+
+  module ContentView
+    module Include
+      def publish_content_view(id, entity_type = :content_views)
+        mapped_api_call entity_type, :publish, {:id => id}
+        rescue => e
+          warn "Publishing of #{to_singular(entity_type)} [#{id}] failed with #{e.class}: #{e.message}"
+      end
+
+      def create_composite_content_view(entity_type, org_id, cv_label, cv_description, cvs)
+        return nil if cvs.empty?
+        if cvs.size == 1
+          return cvs.to_a[0]
+        else
+          # create composite content view
+          cv_versions = []
+          cvs.each do |cv_id|
+            cvvs = list_server_entities(:content_view_versions, {:content_view_id => cv_id})
+            cv_versions << cvvs.collect { |cv| cv['id'] }.max
+          end
+          cv = lookup_entity_in_cache(entity_type, 'label' => cv_label)
+          if cv
+            info "  Content view #{cv_label} already created, reusing."
+          else
+            # create composite content view
+            # for activation key purposes
+            cv = create_entity(
+              entity_type,
+              {
+                :organization_id => org_id,
+                :name => cv_label,
+                :label => cv_label,
+                :composite => true,
+                :description => cv_description,
+                :component_ids => cv_versions
+              },
+              cv_label)
+            # publish the content view
+            info "  Publishing content view: #{cv['id']}"
+            publish_content_view(cv['id'], entity_type)
+          end
+          return cv['id']
+        end
+      end
+
+      # use entity_type as parameter to be able to re-use the method for
+      # :content_views, :ak_content_views, :redhat_content_views, ...
+      def delete_content_view(cv_id, entity_type = :content_views)
+        raise "delete_content_view with #{entity_type}" unless map_target_entity[entity_type] == :content_views
+
+        content_view = get_cache(entity_type)[cv_id]
+        # first delete the content view from associated environments
+        cves = content_view['environments'].collect { |e| e['id'] }
+        cves.each do |cve|
+          begin
+            task = mapped_api_call(
+              entity_type,
+              :remove_from_environment,
+              {
+                :id => content_view['id'],
+                :environment_id => cve
+              })
+          rescue => e
+            warn "Failed to remove content view [#{cv_id}] from environment #{cve} with #{e.class}: #{e.message}"
+          end
+          wait_for_task(task['id'], 1, 0)
+        end
+
+        if content_view['versions'] && !content_view['versions'].empty?
+          cv_version_ids = content_view['versions'].collect { |v| v['id'] }
+
+          begin
+            task = mapped_api_call(
+              entity_type,
+              :remove,
+              {
+                :id => content_view['id'],
+                :content_view_version_ids => cv_version_ids
+              })
+
+            wait_for_task(task['id'], 1, 0)
+          rescue => e
+            warn "Failed to remove versions of content view [#{cv_id}] with #{e.class}: #{e.message}"
+          end
+        else
+          debug "No versions found for #{to_singular(entity_type)} #{cv_id}"
+        end
+
+        delete_entity_by_import_id(entity_type, content_view['id'])
+      end
+    end
+  end
+
+  module ImportLogging
+    module Extend
+      def add_logging_options
+        # Logging options
+        # quiet = go to logfile only
+        # verbose = all output goes to STDOUT as well as log
+        # debug = enable debug-output
+        # default = no debug, only PROGRESS-and-above to STDOUT
+        option ['--quiet'], :flag, 'Be silent - no output to STDOUT', :default => false
+        option ['--debug'], :flag, 'Turn on debugging-information', :default => false
+        option ['--verbose'],
+               :flag,
+               'Be noisy - everything goes to STDOUT and to a logfile',
+               :default => false
+        option ['--logfile'],
+               'LOGFILE',
+               'Where output is logged to',
+               :default => File.expand_path('~/import.log')
+      end
+    end
+
+    module Include
+      def setup_logging
+        @curr_lvl = Logger::INFO
+        @curr_lvl = Logger::DEBUG if option_debug?
+
+        @logger = Logger.new(File.new(option_logfile, 'a'))
+        @logger.level = @curr_lvl
+      end
+
+      def debug(s)
+        log(Logger::DEBUG, s)
+      end
+
+      def info(s)
+        log(Logger::INFO, s)
+      end
+
+      def progress(s)
+        log(Logger::INFO, s, true)
+      end
+
+      def warn(s)
+        log(Logger::WARN, s)
+      end
+
+      def error(s)
+        log(Logger::ERROR, s)
+      end
+
+      def fatal(s)
+        log(Logger::FATAL, s)
+      end
+
+      def logtrace(e)
+        @logger.log(Logger::ERROR, (e.backtrace.join "\n"))
+      end
+
+      def log(lvl, s, always = false)
+        @logger.log(lvl, s)
+        return if option_quiet?
+
+        if always
+          puts s
+        elsif option_verbose?
+          puts s if lvl >= @curr_lvl
+        else
+          puts s if lvl > @curr_lvl
+        end
+      end
+    end
+  end
+
+  module Exceptional
+    module Include
+      def handle_missing_and_supress(what, &block)
+        block.call
+      rescue HammerCLIImport::MissingObjectError => moe
+        error moe.message
+      rescue => e
+        error "Caught #{e.class}:#{e.message} while #{what}"
+        logtrace e
+      end
+
+      # this method catches everything sent to stdout and stderr
+      # and disallows any summary changes
+      #
+      # this is a bad hack, but we need it, as sat6 cannot tell,
+      # whether there're still content hosts associated with a content view
+      # so we try to delete system content views silently
+      def silently(&block)
+        summary_backup = @summary.clone
+        $stdout = StringIO.new
+        $stderr = StringIO.new
+        block.call
+        ensure
+          $stdout = STDOUT
+          $stderr = STDERR
+          @summary = summary_backup
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/organization.rb

@@ -0,0 +1,110 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+
+module HammerCLIImport
+  class ImportCommand
+    class OrganizationImportCommand < BaseCommand
+      command_name 'organization'
+      reportname = 'users'
+      desc "Import Organizations (from spacewalk-report #{reportname})."
+
+      option ['--into-org-id'], 'ORG_ID', 'Import all organizations into one specified by id' do |x|
+        Integer(x)
+      end
+
+      # Where do we expect to find manifest-files?
+      #  NOTE: we won't upload manifests if we're doing into-org-id - the expectation is that
+      #  you have already set up your org
+      option ['--upload-manifests-from'],
+             'MANIFEST_DIR',
+             'Upload manifests found at MANIFEST_DIR. Assumes manifest for "ORG NAME" will be of the form ORG_NAME.zip'
+
+      csv_columns 'organization_id', 'organization'
+
+      persistent_maps :organizations
+
+      def mk_org_hash(data)
+        {
+          :id => data['organization_id'].to_i,
+          :name => data['organization'],
+          :description => "Imported '#{data['organization']}' organization from Red Hat Satellite 5"
+        }
+      end
+
+      # :subscriptions :upload {:org_id => id, :content => File.new(filename, 'rb')}
+      def upload_manifest_for(label, id)
+        # Remember labels we've already processed in this run
+        @manifests ||= []
+        return if @manifests.include? label
+
+        @manifests << label
+        filename = option_upload_manifests_from + '/' + label + '.zip'
+        unless File.exist? filename
+          error "No manifest #{filename} available."
+          return
+        end
+
+        info "Uploading manifest #{filename} to org-id #{id}"
+        manifest_file = File.new(filename, 'rb')
+        request_headers = {:content_type => 'multipart/form-data', :multipart => true}
+
+        rc = api_call :subscriptions, :upload, {:organization_id => id, :content => manifest_file}, request_headers
+        wait_for_task(rc['id'])
+        report_summary :uploaded, :manifests
+      end
+
+      def import_single_row(data)
+        if option_into_org_id
+          unless lookup_entity_in_cache(:organizations, {'id' => option_into_org_id})
+            warn "Organization [#{option_into_org_id}] not found. Skipping."
+            return
+          end
+          map_entity(:organizations, data['organization_id'].to_i, option_into_org_id)
+          return
+        end
+        org = mk_org_hash data
+        new_org = create_entity(:organizations, org, data['organization_id'].to_i)
+        upload_manifest_for(new_org['label'], new_org['id']) unless option_upload_manifests_from.nil?
+      end
+
+      def delete_single_row(data)
+        org_id = data['organization_id'].to_i
+        unless @pm[:organizations][org_id]
+          warn "#{to_singular(:organizations).capitalize} with id #{org_id} wasn't imported. Skipping deletion."
+          return
+        end
+        target_org_id = get_translated_id(:organizations, org_id)
+        if last_in_cache?(:organizations, target_org_id)
+          warn "Won't delete last organization [#{target_org_id}]. Unmapping only."
+          unmap_entity(:organizations, target_org_id)
+          return
+        end
+        if target_org_id == 1
+          warn "Won't delete organization with id [#{target_org_id}]. Unmapping only."
+          unmap_entity(:organizations, target_org_id)
+          return
+        end
+        delete_entity(:organizations, org_id)
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/persistentmap.rb

@@ -0,0 +1,225 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'csv'
+require 'set'
+
+module PersistentMap
+  class PersistentMapError < RuntimeError
+  end
+
+  class << Fixnum
+    def from_s(x)
+      Integer(x) rescue 0
+    end
+  end
+
+  class << String
+    def from_s(x)
+      x
+    end
+  end
+
+  class << self
+    def definitions
+      return @definitions if @definitions
+      @definitions = {}
+
+      [:content_views, :host_collections, :organizations, :repositories, :users].each do |symbol|
+        @definitions[symbol] = ['sat5' => Fixnum], ['sat6' => Fixnum], symbol
+      end
+
+      @definitions[:activation_keys] = ['org_id' => String], ['sat6' => Fixnum], :activation_keys
+      @definitions[:ak_content_views] = ['ak_id' => String], ['sat6' => Fixnum], :content_views
+      @definitions[:system_content_views] = ['ch_seq' => String], ['sat6' => Fixnum], :content_views
+      @definitions[:local_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum], :repositories
+      @definitions[:products] = [{'org_id' => Fixnum}, {'label' => String}], ['sat6' => Fixnum], :products
+      @definitions[:puppet_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}],
+                                           ['sat6' => Fixnum], :repositories
+      @definitions[:redhat_content_views] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum],
+                                            :content_views
+      @definitions[:redhat_repositories] = [{'org_id' => Fixnum}, {'channel_id' => Fixnum}], ['sat6' => Fixnum],
+                                           :repositories
+      @definitions[:systems] = ['sat5' => Fixnum], ['sat6' => String], :systems
+      @definitions[:template_snippets] = ['id' => Fixnum], ['sat6' => Fixnum], :config_templates
+
+      @definitions.freeze
+    end
+  end
+
+  module Extend
+    attr_reader :maps, :map_description, :map_target_entity
+
+    def persistent_map(symbol)
+      defs = PersistentMap.definitions
+
+      raise PersistentMapError, "Unknown persistent map: #{symbol}" unless defs.key? symbol
+
+      # Names of persistent maps
+      @maps ||= []
+      @maps.push symbol
+
+      key_spec, val_spec, target_entity = defs[symbol]
+
+      # Which entities they are mapped to?
+      # Usually they are mapped to the same entities on Sat6 (speaking of api)
+      # But sometimes you need to create same type of Sat6 entities based on
+      # different Sat5 entities, and then it is time for this extra option.
+      @map_target_entity ||= {}
+      @map_target_entity[symbol] = target_entity
+
+      # How keys and values looks like (so they can be nicely stored)
+      @map_description ||= {}
+      @map_description[symbol] = [key_spec, val_spec]
+    end
+
+    def persistent_maps(*list)
+      raise PersistentMapError, 'Persistent maps should be declared only once' if @maps
+      list.each do |map_sym|
+        persistent_map map_sym
+      end
+    end
+  end
+
+  module Include
+    def maps
+      self.class.maps
+    end
+
+    def map_target_entity
+      self.class.map_target_entity
+    end
+
+    def load_persistent_maps
+      @pm = {}
+      maps.each do |map_sym|
+        hash = {}
+        Dir[File.join data_dir, "#{map_sym}-*.csv"].sort.each do |filename|
+          reader = CSV.open(filename, 'r')
+          header = reader.shift
+          raise PersistentMapError, "Importing :#{map_sym} from file #{filename}" unless header == (pm_csv_headers map_sym)
+          reader.each do |row|
+            key, value = pm_decode_row map_sym, row
+            delkey = row[-1] == '-'
+            if delkey
+              hash.delete key
+            else
+              hash[key] = value
+            end
+          end
+        end
+        @pm[map_sym] = add_checks(DeltaHash[hash], self.class.map_description[map_sym], map_sym)
+      end
+    end
+
+    def save_persistent_maps
+      maps.each do |map_sym|
+        next unless @pm[map_sym].changed?
+        CSV.open((File.join data_dir, "#{map_sym}-#{Time.now.utc.iso8601}.csv"), 'wb') do |csv|
+          csv << (pm_csv_headers map_sym)
+          @pm[map_sym].new.each do |key, value|
+            key = [key] unless key.is_a? Array
+            value = [value] unless value.is_a? Array
+            csv << key + value + [nil]
+          end
+          delval = [nil] * (val_arity map_sym)
+          @pm[map_sym].del.each do |key|
+            key = [key] unless key.is_a? Array
+            csv << key + delval + ['-']
+          end
+        end
+      end
+    end
+
+    # Consider entities deleted if they are not present in cache
+    def prune_persistent_maps(cache)
+      maps.each do |map_sym|
+        entity_ids = cache[map_target_entity[map_sym]].keys
+        pm_hash = @pm[map_sym].to_hash
+        extra = pm_hash.values.to_set - entity_ids.to_set
+
+        next if extra.empty?
+
+        debug "Removing #{map_sym} from persistent map: #{extra.to_a.join(' ')}"
+        pm_hash.each do |key, value|
+          @pm[map_sym].delete key if extra.include? value
+        end
+      end
+    end
+
+    private
+
+    # Protective black magic.
+    # Checks whether given values and keys matches description
+    # at the moment of insertion...
+    def add_checks(hash, kv_desc, map_sym)
+      hash.instance_eval do
+        ks, vs = kv_desc
+        @key_desc = ks.collect(&:values) .flatten
+        @val_desc = vs.collect(&:values) .flatten
+        @map_sym = map_sym
+      end
+      class << hash
+        def []=(ks, vs)
+          key = ks
+          val = vs
+          key = [key] unless key.is_a? Array
+          val = [val] unless val.is_a? Array
+          raise "Bad key for persistent map #{@map_sym}: (#{key.inspect} - #{@key_desc.inspect})" \
+            unless key.size == @key_desc.size && key.zip(@key_desc).all? { |k, d| k.is_a? d }
+          raise "Bad value for persistent map #{@map_sym}: (#{val.inspect} - #{@val_desc.inspect}" \
+            unless val.size == @val_desc.size && val.zip(@val_desc).all? { |v, d| v.is_a? d }
+          super
+        end
+      end
+      hash
+    end
+
+    def pm_decode_row(map_sym, row)
+      key_spec, val_spec = self.class.map_description[map_sym]
+      key = []
+      value = []
+
+      key_spec.each do |spec|
+        x = row.shift
+        key.push(spec.values.first.from_s x)
+      end
+
+      val_spec.each do |spec|
+        x = row.shift
+        value.push(spec.values.first.from_s x)
+      end
+
+      key = key[0] if key.size == 1
+      value = value[0] if value.size == 1
+      [key, value]
+    end
+
+    def pm_csv_headers(symbol)
+      key_spec, val_spec = self.class.map_description[symbol]
+      (key_spec + val_spec).collect { |x| x.keys[0] } + ['delete']
+    end
+
+    def val_arity(symbol)
+      _key_spec, val_spec = self.class.map_description[symbol]
+      val_spec.size
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby