hammer_cli_import 0.10.21

data/lib/hammer_cli_import/repository.rb

@@ -0,0 +1,91 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'uri'
+
+module HammerCLIImport
+  class ImportCommand
+    class RepositoryImportCommand < BaseCommand
+      extend ImportTools::Repository::Extend
+      include ImportTools::Repository::Include
+
+      command_name 'repository'
+      reportname = 'repositories'
+      desc "Import repositories (from spacewalk-report #{reportname})."
+
+      csv_columns 'id', 'org_id', 'repo_label', 'source_url', 'repo_type'
+
+      persistent_maps :organizations, :repositories, :products
+
+      add_repo_options
+
+      def mk_product_hash(data, product_name)
+        {
+          :name => product_name,
+          :organization_id => get_translated_id(:organizations, data['org_id'].to_i)
+        }
+      end
+
+      def mk_repo_hash(data, product_id)
+        {
+          :name => data['repo_label'],
+          :product_id => product_id,
+          :url => data['source_url'],
+          :content_type => data['repo_type']
+        }
+      end
+
+      def import_single_row(data)
+        begin
+          product_name = URI.parse(data['source_url']).host.split('.').last(2).join('.').upcase
+        rescue
+          error 'Skipping ' + data['repo_label'] + ' ' + to_singular(:repositories) + ' import, invalid source_url.'
+          report_summary :skipped, :repositories
+          return
+        end
+        product_hash = mk_product_hash(data, product_name)
+        composite_id = [data['org_id'].to_i, product_name]
+        product_id = create_entity(:products, product_hash, composite_id)['id']
+        repo_hash = mk_repo_hash data, product_id
+
+        repo = create_entity(:repositories, repo_hash, data['id'].to_i)
+        with_synced_repo(repo)
+      end
+
+      def delete_single_row(data)
+        # check just becasue we're calling get_translated_id
+        unless @pm[:repositories][data['id'].to_i]
+          info to_singular(:repositories).capitalize + ' with id ' + data['id'] + " wasn't imported. Skipping deletion."
+          return
+        end
+        # find out product id
+        repo_id = get_translated_id(:repositories, data['id'].to_i)
+        product_id = lookup_entity(:repositories, repo_id)['product']['id']
+        # delete repo
+        delete_entity(:repositories, data['id'].to_i)
+        # delete its product, if it's not associated with any other repositories
+        product = lookup_entity(:products, product_id, true)
+
+        delete_entity_by_import_id(:products, product_id) if product['repository_count'] == 0
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/repositoryenable.rb

@@ -0,0 +1,250 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'hammer_cli_import'
+require 'json'
+
+module HammerCLIImport
+  class ImportCommand
+    class RepositoryEnableCommand < BaseCommand
+      extend ImportTools::Repository::Extend
+      include ImportTools::Repository::Include
+      include ImportTools::ContentView::Include
+
+      command_name 'repository-enable'
+      reportname = 'channels'
+      desc "Enable any Red Hat repositories accessible to any Organization (from spacewalk-report #{reportname})."
+
+      option ['--repository-map'],
+             'FILE_NAME',
+             'JSON file mapping channel-labels to repository information',
+             :default => File.expand_path('../../channel_data_pretty.json', File.dirname(__FILE__)) do |filename|
+        raise ArgumentError, "Channel-to-repository-map file #{filename} does not exist" unless File.exist? filename
+        filename
+      end
+
+      option ['--dry-run'],
+             :flag,
+             'Only show the repositories that would be enabled',
+             :default => false
+
+      add_repo_options
+
+      # Required or BaseCommand gets angry at you
+      csv_columns 'channel_id', 'channel_label', 'channel_name', 'number_of_packages', 'org_id'
+      persistent_maps :organizations, :products,
+                      :redhat_repositories, :redhat_content_views, :repositories
+
+      # BaseCommand will read our channel-csv for us
+      def import_single_row(row)
+        handle_row(row, true)
+      end
+
+      def delete_single_row(row)
+        handle_row(row, false)
+      end
+
+      def handle_row(row, enable)
+        if row['org_id'] # Not a Red Hat channel
+          info " Skipping #{row['channel_label']} in organization #{row['org_id']}"
+          return
+        end
+
+        # read_channel_mapping_data will be called only once per subcommand
+        @channel_to_repo ||= read_channel_mapping_data(option_repository_map)
+        channel_label = row['channel_label']
+        channel_id = row['channel_id'].to_i
+        repo_set_info = @channel_to_repo[channel_label]
+
+        if repo_set_info.nil? # not mapped channel (like proxy)
+          info " Skipping nontransferable #{row['channel_label']}"
+          return
+        end
+
+        # rely on we see only products in imported organizations
+        get_cache(:products).each do |product_id, product|
+          rsets = list_server_entities(:repository_sets, {:product_id => product_id}, true)
+
+          rsets.each do |rs|
+            next if repo_set_info['set-url'] != rs['contentUrl'] &&
+                    repo_set_info['set-url'].gsub('$releasever', repo_set_info['version']) != rs['contentUrl']
+
+            product_org = lookup_entity_in_cache(:organizations, {'label' => product['organization']['label']})
+            composite_rhcv_id = [get_original_id(:organizations, product_org['id']), channel_id]
+            if enable
+              # Turn on the specific repository
+              rh_repo = enable_repos(product_org, product_id, rs['id'], repo_set_info, row)
+              next if rh_repo.nil? || option_dry_run?
+
+              # Finally, if requested, kick off a sync
+              with_synced_repo rh_repo do
+                cv = create_entity(
+                  :redhat_content_views,
+                  {
+                    :organization_id => product_org['id'],
+                    :name => row['channel_name'],
+                    :description => 'Red Hat channel migrated from Satellite 5',
+                    :repository_ids  => [rh_repo['id']]
+                  },
+                  composite_rhcv_id)
+                begin
+                  publish_content_view(cv['id'], :redhat_content_views)
+                rescue RestClient::Exception => e
+                  msg = JSON.parse(e.response)['displayMessage']
+                  error "#{e.http_code} trying to publish content-view #{row['channel_name']} :\n #{msg}\n"
+                  next
+                end
+              end
+            else
+              if @pm[:redhat_content_views][composite_rhcv_id]
+                delete_content_view(get_translated_id(:redhat_content_views, composite_rhcv_id), :redhat_content_views)
+              end
+              disable_repos(product_org, product_id, rs['id'], repo_set_info, channel_label)
+            end
+          end
+        end
+      end
+
+      # Hydrate the channel-to-repository-data mapping struct
+      def read_channel_mapping_data(filename)
+        channel_map = {}
+
+        File.open(filename, 'r') do |f|
+          json = f.read
+          channel_map = JSON.parse(json)
+        end
+        return channel_map
+      end
+
+      # this way we're able to get from the server, what repositories within a repository-set are enabled
+      def find_repo_in_reposet(product_id, repo_set_id, info)
+        repos = api_call(
+          :repository_sets,
+          :available_repositories,
+          'product_id' => product_id,
+          'id' => repo_set_id)
+
+        if repos['results'][0] && repos['results'][0]['substitutions'].size == 1
+          return lookup_entity_in_array(repos['results'],
+                                        {'substitutions' => {'basearch' => info['arch']}})
+        end
+        return lookup_entity_in_array(repos['results'],
+                                      {'substitutions' => {'basearch' => info['arch'], 'releasever' => info['version']}})
+      end
+
+      # and this is quite a pain - to get real ids of the enabled repositories
+      # search according to the repository name
+      def find_enabled_repo(product_id, repo_set_id, repo_name)
+        reposet = api_call(
+          :repository_sets,
+          :show,
+          'product_id' => product_id,
+          'id' => repo_set_id)
+
+        enabled_repo = lookup_entity_in_array(reposet['repositories'], {'name' => repo_name})
+        info "Repository '#{repo_name}' already enabled as #{enabled_repo['id']}."
+        return lookup_entity(:redhat_repositories, enabled_repo['id'])
+      end
+
+      # Given a repository-set and a channel-to-repo info for that channel,
+      # enable the correct repository
+      def enable_repos(org, prod_id, repo_set_id, info, row)
+        channel_label = row['channel_label']
+        channel_id = row ['channel_id'].to_i
+        repo = find_repo_in_reposet(prod_id, repo_set_id, info)
+        if repo.nil?
+          warn "Repository #{info['url']} for (#{info['arch']} x #{info['version']}) not found!"
+          return
+        end
+
+        return find_enabled_repo(prod_id, repo_set_id, repo['repo_name']) if repo['enabled']
+
+        info "Enabling #{info['url']} for channel #{channel_label} in org #{org['id']}"
+        begin
+          unless option_dry_run?
+            rc = api_call(
+              :repository_sets,
+              :enable,
+              'product_id' => prod_id,
+              'id' => repo_set_id,
+              'basearch' => repo['substitutions'].key?('basearch') ? info['arch'] : '',
+              'releasever' => repo['substitutions'].key?('releasever') ? info['version'] : '')
+
+            original_org_id = get_original_id(:organizations, org['id'])
+            map_entity(:redhat_repositories, [original_org_id, channel_id], rc['input']['repository']['id'])
+            # store to cache (using lookup_entity, because :redhat_repositories api
+            # does not return full repository hash)
+            return lookup_entity(:redhat_repositories, rc['input']['repository']['id'], true)
+          end
+        rescue RestClient::Exception  => e
+          if e.http_code == 409
+            info '...already enabled.'
+            return find_enabled_repo(prod_id, repo_set_id, repo['repo_name'])
+          else
+            error "...repository enablement failed with error '#{e.http_code}, #{e.message}' - skipping."
+          end
+        end
+      end
+
+      def disable_repos(org, prod_id, repo_set_id, info, channel_label)
+        repo = lookup_entity_in_cache(
+          :redhat_repositories,
+          {
+            'content_id' => repo_set_id,
+            'organization' => {'label' => org['label']}
+          })
+        # find the repo from reposet to get also the substitutions
+        # otherwise it's not possible to disable certain repositories
+        repo = find_repo_in_reposet(prod_id, repo_set_id, info) if repo
+        unless repo
+          error "Unknown repository (#{channel_label} equivalent) to disable."
+          return
+        end
+        info "Disabling #{info['url']} for channel #{channel_label} in org #{org['id']}"
+        begin
+          unless option_dry_run?
+            rc = api_call(
+              :repository_sets,
+              :disable,
+              'product_id' => prod_id,
+              'id' => repo_set_id,
+              'basearch' => repo['substitutions'].key?('basearch') ? info['arch'] : '',
+              'releasever' => repo['substitutions'].key?('releasever') ? info['version'] : '')
+
+            unmap_entity(:redhat_repositories, rc['input']['repository']['id'])
+            get_cache(:redhat_repositories).delete(rc['input']['repository']['id'])
+            return rc['input']['repository']
+          end
+        rescue RestClient::Exception  => e
+          if e.http_code == 404
+            error '...no such repository to disable.'
+          else
+            error "...repository disable failed with error '#{e.http_code}, #{e.message}' - skipping."
+          end
+        end
+      end
+
+      def post_import(_file)
+        HammerCLI::EX_OK
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/templatesnippet.rb

@@ -0,0 +1,67 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'apipie-bindings'
+
+module HammerCLIImport
+  class ImportCommand
+    class TemplateSnippetImportCommand < BaseCommand
+      command_name 'template-snippet'
+      reportname = 'kickstart-scripts'
+      desc "Import template snippets (from spacewalk-report #{reportname})."
+
+      csv_columns \
+        'id', 'org_id', 'script_name', 'kickstart_label', 'position',
+        'script_type', 'chroot', 'interpreter', 'data'
+
+      persistent_maps :organizations, :template_snippets
+
+      def mk_snippet_hash(data)
+        template = "%#{data['script_type']}"
+        template += ' --nochroot' if data['chroot'] == 'N'
+        template += " --interpreter #{data['interpreter']}" if data['interpreter']
+        template += "\n"
+        template += data['data'] if data['data']
+        template += "\n" unless template.end_with? "\n"
+        template += "%end\n"
+        {
+          :name => "#{data['kickstart_label']}-#{data['org_id']}-" \
+          "#{data['position']}-#{data['script_name']}-#{data['script_type']}",
+          :template => template,
+          # nowadays templates do not get associated with an organization
+          # :organization_id => get_translated_id(:organizations, data['org_id'].to_i),
+          :snippet => true,
+          # audit_comment does not get stored anyway
+          :audit_comment => ''
+        }
+      end
+
+      def import_single_row(data)
+        snippet = mk_snippet_hash data
+        create_entity(:template_snippets, snippet, data['id'].to_i)
+      end
+
+      def delete_single_row(data)
+        delete_entity(:template_snippets, data['id'].to_i)
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/user.rb

@@ -0,0 +1,155 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+require 'yaml'
+
+module HammerCLIImport
+  class ImportCommand
+    class UserImportCommand < BaseCommand
+      command_name 'user'
+      reportname = 'users'
+      desc "Import Users (from spacewalk-report #{reportname})."
+
+      option ['--new-passwords'], 'FILE_NAME', 'Output for new passwords' do |filename|
+        raise ArgumentError, "File #{filename} already exists" if File.exist? filename
+        filename
+      end
+
+      option ['--merge-users'], :flag, 'Merge pre-created users (except admin)', :default => false
+
+      option ['--role-mapping'], 'FILE_NAME',
+             'Mapping of Satellite-5 role names to Satellite-6 defined roles',
+             :default => '/etc/hammer/cli.modules.d/import/role_map.yml'
+
+      validate_options do
+        any(:option_new_passwords, :option_delete).required
+      end
+
+      csv_columns 'organization_id', 'user_id', 'username',\
+                  'last_name', 'first_name', 'email', 'role', 'active'
+
+      persistent_maps :organizations, :users
+
+      # Override so we can read the role-map *once*, not *once per user*
+      def first_time_only
+        if File.exist? option_role_mapping
+          @role_map = YAML.load_file(option_role_mapping)
+        else
+          warn "Role-mapping file #{option_role_mapping} not found, no roles will be assigned"
+        end
+        return 'loaded'
+      end
+
+      def genpw(username)
+        username + '_' + (0...8).collect { ('a'..'z').to_a[rand(26)] }.join
+      end
+
+      # Admin-flag should be set if any Sat5-role has '_admin_' in its map
+      def admin?(data)
+        return false if @role_map.nil?
+
+        roles = split_multival(data['role'], false)
+        is_admin = false
+        roles.each do |r|
+          is_admin ||= (@role_map[r.gsub(' ', '-')].include? '_admin_')
+        end
+        return is_admin
+      end
+
+      # Return list-of-role-ids that match any sat5-role associated with this user
+      # XXX: Role-api doesn't return what it needs to
+      # Until BZ 1102816 is fixed, this doesn't work
+      # It does serve to show the infrastructure/approach required
+      def role_ids_for(data)
+        role_list = []
+        return role_list if @role_map.nil?
+
+        users_roles = split_multival(data['role'], false)
+        fm_roles = api_call(:roles, :index, 'per_page' => 999999)['results']
+        debug fm_roles.inspect
+        users_roles.each do |s5r|
+          fm_roles.each do |fr|
+            role_list << fr['id'] if @role_map[s5r.gsub(' ', '-')].include? fr['name']
+          end
+        end
+
+        return role_list
+      end
+
+      def mk_user_hash(data)
+        username = data['username']
+        username = 'sat5_admin' if username == 'admin'
+        {
+          :login => username,
+          :firstname => data['first_name'],
+          :lastname => data['last_name'],
+          :mail => data['email'],
+          :auth_source_id => 1,
+          :password => genpw(username),
+          :organization_ids => [get_translated_id(:organizations, data['organization_id'].to_i)],
+          :location_ids => [],
+          :admin => admin?(data),
+          :role_ids => role_ids_for(data)
+        }
+      end
+
+      def post_import(_)
+        return if @new_passwords.nil? || @new_passwords.empty?
+        CSVHelper.csv_write_hashes option_new_passwords, [:mail, :login, :password], @new_passwords
+      end
+
+      def import_single_row(data)
+        @first_time ||= first_time_only
+        user = mk_user_hash data
+        new_user = true
+
+        user_id = data['user_id'].to_i
+        login = user[:login]
+
+        unless @pm[:users][user_id].nil?
+          info "User #{login} already imported."
+          return
+        end
+
+        if option_merge_users?
+          existing_user = lookup_entity_in_cache :users, 'login' => user[:login]
+
+          unless existing_user.nil?
+            info "User with login #{login} already exists. Associating..."
+            map_entity :users, user_id, existing_user['id']
+            new_user = false
+          end
+        end
+
+        return unless new_user
+
+        create_entity :users, user, user_id
+
+        @new_passwords ||= []
+        @new_passwords << {:login => user[:login], :password => user[:password], :mail => user[:mail]}
+      end
+
+      def delete_single_row(data)
+        delete_entity(:users, data['user_id'].to_i)
+      end
+    end
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import/version.rb

@@ -0,0 +1,25 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+module HammerCLIImport
+  def self.version
+    @version ||= Gem::Version.new('0.10.21')
+  end
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby

data/lib/hammer_cli_import.rb

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2014 Red Hat Inc.
+#
+# This file is part of hammer-cli-import.
+#
+# hammer-cli-import is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# hammer-cli-import is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with hammer-cli-import.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+require 'hammer_cli'
+
+module HammerCLIImport
+  # def self.exception_handler_class
+  #   HammerCLIImport::ExceptionHandler
+  # end
+
+  require 'hammer_cli_import/asynctasksreactor'
+  require 'hammer_cli_import/csvhelper'
+  require 'hammer_cli_import/deltahash'
+  require 'hammer_cli_import/fixtime'
+  require 'hammer_cli_import/importtools'
+  require 'hammer_cli_import/persistentmap'
+
+  require 'hammer_cli_import/base'
+  require 'hammer_cli_import/import'
+
+  require 'hammer_cli_import/all'
+  require 'hammer_cli_import/activationkey'
+  require 'hammer_cli_import/configfile'
+  require 'hammer_cli_import/contentview'
+  require 'hammer_cli_import/contenthost'
+  require 'hammer_cli_import/hostcollection'
+  require 'hammer_cli_import/organization'
+  require 'hammer_cli_import/repository'
+  require 'hammer_cli_import/repositoryenable'
+  require 'hammer_cli_import/templatesnippet.rb'
+  require 'hammer_cli_import/user'
+  require 'hammer_cli_import/version'
+
+  # This has to be after all subcommands
+  require 'hammer_cli_import/autoload'
+end
+# vim: autoindent tabstop=2 shiftwidth=2 expandtab softtabstop=2 filetype=ruby