hammer_cli_import 0.10.21

data/config/import/config_macros.yml

@@ -0,0 +1,16 @@
+# spacewalk-macro-name : puppet-fact
+rhn.system.sid                                        :
+rhn.system.profile_name                               :
+rhn.system.description                                :
+rhn.system.hostname                                   : '@fqdn'
+rhn.system.ip_address                                 : '@ipaddress'
+rhn.system.ip6_address                                : '@ipaddress6'
+rhn.system.custom_info(key_name)                      :
+rhn.system.net_interface.ip_address(eth_device)       : '@ipaddress_{NETWORK INTERFACE}'
+rhn.system.net_interface.netmask(eth_device)          : '@netmask_{NETWORK INTERFACE}'
+rhn.system.net_interface.ip6_address(eth_device)      : '@ipaddress6_{NETWORK INTERFACE}'
+rhn.system.net_interface.ip6_netmask(eth_device)      : '@netmask_{NETWORK INTERFACE}'
+rhn.system.net_interface.broadcast(eth_device)        :
+rhn.system.net_interface.hardware_address(eth_device) : '@macaddress_{NETWORK INTERFACE}'
+rhn.system.net_interface.driver_module(eth_device)    :
+

data/config/import/interview_answers.yml

@@ -0,0 +1,13 @@
+# interview-question-key : answer
+# special-strings:
+#   #{module_name} == the name of the module being generated
+#
+version     : '1.0.0'
+author      : 'Red Hat'
+license     : 'GPL3'
+summary     : "#{module_name}"
+description : "Module #{module_name} created from Sat5 config-channel"
+srcrepo     : ""
+learnmore   : ""
+fileissues  : ""
+

data/config/import/role_map.yml

@@ -0,0 +1,10 @@
+# sat5-role-label,sat6-role-name-list
+# _admin_ means "flag any user with this role as an Administrator"
+#
+Satellite-Administrator      : [ _admin_ ]
+Organization-Administrator   : [ _admin_ ]
+Channel-Administrator        : [ ]
+Configuration-Administrator  : [ ]
+Monitoring-Administrator     : [ ]
+System-Group-Administrator   : [ ]
+Activation-Key-Administrator : [ ]

data/config/import.yml

@@ -0,0 +1,2 @@
+:import:
+    :enable_module: true

data/lib/hammer_cli_import/activationkey.rb

@@ -0,0 +1,156 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'apipie-bindings'
+
+module HammerCLIImport
+  class ImportCommand
+    class ActivationKeyImportCommand < BaseCommand
+      include ImportTools::ContentView::Include
+      include ImportTools::LifecycleEnvironment::Include
+
+      command_name 'activation-key'
+      reportname = 'activation-keys'
+      desc "Import Activation Keys (from spacewalk-report #{reportname})."
+
+      csv_columns 'token', 'org_id', 'note', 'usage_limit', 'base_channel_id', 'child_channel_id', 'server_group_id'
+
+      persistent_maps :organizations, :host_collections, :content_views, :redhat_content_views,
+                      :ak_content_views, :activation_keys
+
+      def mk_ak_hash(data)
+        usage_limit = 'unlimited'
+        usage_limit = data['usage_limit'].to_i if data['usage_limit']
+        debug "  Activation key usage_limit: #{usage_limit}"
+        {
+          :name => data['token'],
+          :organization_id => get_translated_id(:organizations, data['org_id'].to_i),
+          :label => data['token'],
+          :description => data['note'],
+          :usage_limit => usage_limit
+        }
+      end
+
+      def associate_host_collections(ak_id, server_group_ids)
+        translated_ids = server_group_ids.collect { |sg_id| get_translated_id(:host_collections, sg_id) }
+        info "  Associating activation key [#{ak_id}] with host collections [#{translated_ids.join(', ')}]"
+        api_call(
+          :activation_keys,
+          :add_host_collections,
+          {
+            :id => ak_id,
+            :host_collection_ids => translated_ids
+          })
+      end
+
+      def import_single_row(data)
+        if data['base_channel_id'].nil?
+          # if base channel id is empty,
+          # 'Spacewalk Default' was used on Sat5
+          info "Skipping activation-key #{data['token']}: Migrating activation-keys with " \
+            "'Red Hat Satellite Default' as base channel is not supported."
+          report_summary :skipped, :activation_keys
+          return
+        end
+        ak_hash = mk_ak_hash data
+        ak = create_entity(:activation_keys, ak_hash, data['token'])
+        if (data['server_group_id'])
+          associate_host_collections(ak['id'], split_multival(data['server_group_id']))
+        end
+        @ak_content_views ||= {}
+        @ak_content_views[ak['id'].to_i] ||= Set.new
+        if data['base_channel_id']
+          split_multival(data['base_channel_id']).each do |base_channel_id|
+            @ak_content_views[ak['id'].to_i] << begin
+              get_translated_id(:redhat_content_views, [data['org_id'].to_i, base_channel_id])
+            rescue HammerCLIImport::MissingObjectError
+              begin
+                get_translated_id(:content_views, base_channel_id)
+              rescue HammerCLIImport::MissingObjectError
+                error "Can't find content view for channel ID [#{base_channel_id}] for key [#{data['token']}]"
+              end
+            end
+          end
+        else
+          # if base channel id is empty,
+          # 'Spacewalk Default' was used on Sat5
+          # Since we can not migrate them due to
+          # bug 1126924, we skip it right at the beggining
+          # of this function.
+          debug ' Red Hat Satellite Default activation keys are not supported.'
+        end
+        split_multival(data['child_channel_id']).each do |child_ch|
+          @ak_content_views[ak['id'].to_i] << begin
+            get_translated_id(:redhat_content_views, [data['org_id'].to_i, child_ch])
+          rescue HammerCLIImport::MissingObjectError
+            begin
+              get_translated_id(:content_views, child_ch)
+            rescue HammerCLIImport::MissingObjectError
+              error "Can't find content view for channel ID [#{child_ch}] for key [#{data['token']}]"
+            end
+          end
+        end
+      end
+
+      def post_import(_csv_file)
+        return unless @ak_content_views
+        @ak_content_views.each do |ak_id, cvs|
+          if cvs.include? nil
+            warn "Skipping content view association for activation key [#{ak_id}]. Dependent content views not ready."
+            next
+          end
+          handle_missing_and_supress "processing activation key #{ak_id}" do
+            ak = lookup_entity(:activation_keys, ak_id)
+            ak_cv_hash = {}
+            org_id = lookup_entity_in_cache(:organizations, {'label' => ak['organization']['label']})['id']
+            ak_cv_hash[:content_view_id] = create_composite_content_view(
+              :ak_content_views,
+              org_id,
+              "ak_#{ak_id}",
+              "Composite content view for activation key #{ak['name']}",
+              cvs)
+            ak_cv_hash[:environment_id] = get_env(org_id, 'Library')['id']
+            ak_cv_hash[:organization_id] = org_id
+            if ak_cv_hash[:content_view_id]
+              info "  Associating activation key [#{ak_id}] with content view [#{ak_cv_hash[:content_view_id]}]"
+              # associate the content view with the activation key
+              update_entity(:activation_keys, ak_id, ak_cv_hash)
+            else
+              info '  Skipping content-view associations.'
+            end
+          end
+        end
+      end
+
+      def delete_single_row(data)
+        unless @pm[:activation_keys][data['token']]
+          info to_singular(:activation_keys).capitalize + ' with id ' + data['token'] +
+            " wasn't imported. Skipping deletion."
+          return
+        end
+        ak = @cache[:activation_keys][get_translated_id(:activation_keys, data['token'])]
+        delete_entity(:activation_keys, data['token'])
+        delete_content_view(ak['content_view']['id'].to_i, :ak_content_views) if
+          ak['content_view'] && was_translated(:ak_content_views, ak['content_view']['id'])
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/all.rb

@@ -0,0 +1,253 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'hammer_cli_import'
+
+module HammerCLIImport
+  class ImportCommand
+    class AllCommand < HammerCLI::AbstractCommand
+      extend ImportTools::Repository::Extend
+      extend ImportTools::ImportLogging::Extend
+      extend AsyncTasksReactor::Extend
+      include ImportTools::ImportLogging::Include
+
+      command_name 'all'
+      desc 'Load ALL data from a specified directory that is in spacewalk-export format.'
+
+      option ['--directory'], 'DIR_PATH', 'stargate-export directory', :default => '/tmp/exports'
+      option ['--delete'], :flag, 'Delete entities instead of importing them', :default => false
+      option ['--macro_mapping'], 'FILE', 'Mapping of Satellite-5 config-file-macros to puppet facts'
+      option ['--manifest-directory'], 'DIR_PATH', 'Directory holding manifests'
+      option ['--entities'], 'entity[,entity...]', 'Import specific entities', :default => 'all'
+      option ['--list-entities'], :flag, 'List entities we understand', :default => false
+      option ['--into-org-id'], 'ORG_ID', 'Import all organizations into one specified by id'
+      option ['--merge-users'], :flag, 'Merge pre-created users (except admin)', :default => false
+      option ['--dry-run'], :flag, 'Show what we would have done, if we\'d been allowed', :default => false
+
+      add_repo_options
+      add_logging_options
+
+      # An ordered-list of the entities we know how to import
+      class << self; attr_accessor :entity_order end
+      @entity_order = %w(organization user host-collection repository-enable repository
+                         content-view activation-key template-snippet config-file content-host)
+
+      #
+      # A list of what we know how to do.
+      # The map has entries of
+      #   import-entity => {sat5-export-name, import-classname, entities-we-are-dependent-on, should-import}
+      # The code will look for classes HammerCLIImport::ImportCommand::<import-classname>
+      # It will look in ~/exports/<Sat5-export-name>.csv for data
+      #
+      class << self; attr_accessor :known end
+      @known = {
+        'activation-key' =>
+                    {'export-file' => 'activation-keys',
+                     'import-class' => 'ActivationKeyImportCommand',
+                     'depends-on' => 'organization',
+                     'import' => false },
+        'config-file' =>
+                    {'export-file' => 'config-files-latest',
+                     'import-class' => 'ConfigFileImportCommand',
+                     'depends-on' => 'organization',
+                     'import' => false },
+        'content-host' =>
+                    {'export-file' => 'system-profiles',
+                     'import-class' => 'ContentHostImportCommand',
+                     'depends-on' => 'content-view,host-collection,repository,organization',
+                     'import' => false },
+        'content-view' =>
+                    {'export-file' => 'CHANNELS/export',
+                     'import-class' => 'LocalRepositoryImportCommand',
+                     'depends-on' => 'repository,organization',
+                     'import' => false },
+        'repository' =>
+                    {'export-file' => 'repositories',
+                     'import-class' => 'RepositoryImportCommand',
+                     'depends-on' => 'organization',
+                     'import' => false },
+        'host-collection' =>
+                    {'export-file' => 'system-groups',
+                     'import-class' => 'SystemGroupImportCommand',
+                     'depends-on' => 'organization',
+                     'import' => false },
+        'organization' =>
+                    {'export-file' => 'users',
+                     'import-class' => 'OrganizationImportCommand',
+                     'depends-on' => '',
+                     'import' => false },
+        'repository-enable' =>
+                    {'export-file' => 'channels',
+                     'import-class' => 'RepositoryEnableCommand',
+                     'depends-on' => 'organization',
+                     'import' => false },
+        'template-snippet' =>
+                    {'export-file' => 'kickstart-scripts',
+                     'import-class' => 'TemplateSnippetImportCommand',
+                     'import' => false },
+        'user' =>
+                    {'export-file' => 'users',
+                     'import-class' => 'UserImportCommand',
+                     'depends-on' => 'organization',
+                     'import' => false }
+      }
+
+      def do_list
+        puts 'Entities I understand:'
+        AllCommand.entity_order.each do |an_entity|
+          puts "  #{an_entity}"
+        end
+      end
+
+      # What are we being asked to import?
+      # Marks what we asked for, and whatever those things are dependent on, to import
+      def set_import_targets
+        to_import = option_entities.split(',')
+        AllCommand.known.each_key do |key|
+          AllCommand.known[key]['import'] = (to_import.include?(key) || to_import.include?('all'))
+          next if AllCommand.known[key]['depends-on'].nil? || !AllCommand.known[key]['import']
+
+          depends_on = AllCommand.known[key]['depends-on'].split(',')
+          depends_on.each do |entity_name|
+            AllCommand.known[entity_name]['import'] = true
+          end
+        end
+      end
+
+      # config-file may need --macro-mapping
+      def config_file_args(args)
+        args << '--macro-mapping' << "#{option_macro_mapping}" unless option_macro_mapping.nil?
+        return args
+      end
+
+      # 'content-host needs --export-directory
+      def content_host_args(args)
+        args << '--export-directory' << File.join(File.expand_path('~'), 'rpm-working-dir')
+        return args
+      end
+
+      # 'content-view needs --dir, knows its own --csv-file in that dir, and may need --no-async
+      def content_view_args(args)
+        args << '--dir' << "#{option_directory}/CHANNELS"
+        args << '--no-async' if option_no_async?
+        args << '--synchronize' if option_synchronize?
+        args << '--wait' if option_wait?
+        return args
+      end
+
+      # 'organization' may need --into-org-id
+      def organization_args(args)
+        args << '--into-org-id' << option_into_org_id unless option_into_org_id.nil?
+        args << '--upload-manifests-from' << option_manifest_directory unless option_manifest_directory.nil?
+        return args
+      end
+
+      # repository and repo-enable may need --synch, --wait, and --no-async
+      def repository_args(args)
+        args << '--synchronize' if option_synchronize?
+        args << '--wait' if option_wait?
+        args << '--no-async' if option_no_async?
+        return args
+      end
+
+      # 'user' needs --new-passwords and may need --merge-users
+      def user_args(args)
+        pwd_filename = "passwords_#{Time.now.utc.iso8601}.csv"
+        args << '--new-passwords' << pwd_filename
+        args << '--merge-users' if option_merge_users?
+        return args
+      end
+
+      # Some subcommands have their own special args
+      # This is the function that will know them all
+      def build_args(key, filename)
+        if key == 'content-view'
+          csv = ['--csv-file', "#{option_directory}/CHANNELS/export.csv"]
+        else
+          csv = ['--csv-file', filename]
+        end
+        return csv << '--delete' if option_delete?
+
+        case key
+        when 'config-file'
+          args = config_file_args(csv)
+        when 'content-host'
+          args = content_host_args(csv)
+        when 'content-view'
+          args = content_view_args(csv)
+        when 'organization'
+          args = organization_args(csv)
+        when 'repository', 'repository-enable'
+          args = repository_args(csv)
+        when 'user'
+          args = user_args(csv)
+        else
+          args = csv
+        end
+        return args
+      end
+
+      # Get entities-to-be-processed, in the right order (reversed if deleting)
+      def entities
+        if option_delete?
+          return AllCommand.entity_order.reverse
+        else
+          return AllCommand.entity_order
+        end
+      end
+
+      # Do the import(s)
+      def import_from
+        entities.each do |key|
+          a_map = AllCommand.known[key]
+          if a_map['import']
+            import_file = "#{option_directory}/#{a_map['export-file']}.csv"
+            args = build_args(key, import_file)
+            if File.exist? import_file
+              progress format('Import %-20s with arguments %s', key, args.join(' '))
+
+              #############################################################
+              # MAGIC! We create a class from the class-name-string here! #
+              #############################################################
+              import_class = HammerCLIImport::ImportCommand.const_get(a_map['import-class'])
+              unless option_dry_run?
+                import_class.new(args).run(args)
+              end
+            else
+              progress "...SKIPPING, no file #{import_file} available."
+            end
+          end
+        end
+      end
+
+      def execute
+        setup_logging
+        if option_list_entities?
+          do_list
+        else
+          set_import_targets
+          import_from
+        end
+        HammerCLI::EX_OK
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/asynctasksreactor.rb

@@ -0,0 +1,187 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# Note to future self (and possibly others): unless there is a
+# (serious) bug here, you probably do not want to modify this code.
+
+#  Main thread                     Thread for async tasks
+#     |                                 |
+#     |  enq task          deq task     |<---.
+#     |'----------> Queue ----------->  |     |
+#     |                                 |'---'
+#     |                                 |
+#     V                                 V
+require 'thread'
+
+module HammerCLIImport
+  # Reactor for async tasks
+  # Include submodule should be included in class that
+  # implements @annotate_tasks@ that takes list
+  # of UUIDS and returns map, annotating every
+  # given UUID with :finished bool and :progress float.
+  module AsyncTasksReactor
+    module Extend
+      def add_async_tasks_reactor_options
+        option ['--no-async'], :flag, 'Wait for async tasks in foreground', :default => false
+      end
+    end
+
+    module Include
+      # Call from init
+      def atr_init
+        # Will create thread on demand
+        @thread = nil
+
+        @mutex = Mutex.new
+        @queue = Queue.new
+        @task_map = {}
+        @thread_finish = false
+        @async_tasks_todo = 0
+        @async_tasks_done = 0
+      end
+
+      # Call to pospone execution of @block@ till all tasks are finished
+      # Never ever use @return@ inside provided do block.
+      def postpone_till(uuids, &block)
+        if option_no_async?
+          wait_for uuids, &block
+          return
+        end
+        if uuids.empty?
+          info 'Nothing to wait for, running in main thread.'
+          block.call
+          return
+        end
+        info "Registering tasks for uuids: #{uuids.inspect}."
+        uuids.sort!
+        @queue.enq([uuids, block])
+        start_async_task_thread
+        nil
+      end
+
+      # Variant for case when we do not want run thing in async
+      def wait_for(uuids, &block)
+        info "Waiting for uuids (non async): #{uuids.inspect}."
+        n = 1
+        loop do
+          annotated = annotate_tasks uuids
+          break if annotated.all? { |_, v| v[:finished] }
+          sleep n
+          n = [n + 1, 10].min
+        end
+        block.call
+      end
+
+      # Has to be called before main thread ends.
+      def atr_exit
+        info 'Waiting for async tasks to finish' unless @task_map.empty?
+        @mutex.synchronize do
+          @thread_finish = true
+          @thread.run
+        end
+        @thread.join
+      rescue NoMethodError
+        nil
+      end
+
+      private
+
+      def add_task(uuids, p)
+        @async_tasks_todo += 1
+        @task_map[uuids] ||= []
+        @task_map[uuids] << p
+      end
+
+      def pick_up_tasks_from_the_queue
+        Thread.stop if @mutex.synchronize do
+          if @task_map.empty? && @queue.empty?
+            if @thread_finish
+              info 'Exiting thread (exit requested, all tasks done).'
+              Thread.exit
+            else
+              true
+            end
+          else
+            false
+          end
+        end
+
+        # Do at most what we have at some point
+        @queue.size.times do
+          begin
+            uuids, p = @queue.deq true
+            add_task uuids, p
+          rescue ThreadError
+            break
+          end
+        end
+      end
+
+      def start_async_task_thread
+        puts 'Starting thread for async tasks' unless @thread
+        @thread ||= Thread.new do
+          loop do
+            some_tasks_done = false
+            pick_up_tasks_from_the_queue
+
+            next if @task_map.empty?
+
+            all_uuids = @task_map.keys.flatten.uniq
+            annotated = annotate_tasks all_uuids
+            finished = []
+            progresses = []
+            annotated.each do |uuid, info|
+              finished << uuid if info[:finished]
+              progresses << info[:progress]
+            end
+            avg = progresses.instance_eval { reduce(0, :+) / size.to_f }
+            progress = format '%5.2f', (avg * 100)
+
+            @task_map.keys.each do |uuids|
+              next unless (uuids - finished).empty?
+              info "Condition #{uuids.inspect} met"
+              @task_map[uuids].each do |task|
+                begin
+                  task.call
+                rescue => e
+                  info "Exception caught while executing post-#{uuids.inspect}:"
+                  info e.inspect
+                  logtrace e
+                end
+                @async_tasks_done += 1
+                some_tasks_done = true
+              end
+              @task_map.delete uuids
+            end
+
+            print = @mutex.synchronize do
+              some_tasks_done || @thread_finish
+            end
+            self.progress "Asynchronous tasks: #{@async_tasks_done} " \
+              "of #{@async_tasks_todo + @queue.size} done (~#{progress}%)" \
+              if print
+
+            sleep 1 unless @task_map.empty?
+          end
+        end
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/autoload.rb

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# This has to be included after all other subcommands were loaded
+# to work properly.
+module HammerCLIImport
+  class ImportCommand
+    autoload_subcommands
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby