3scale_toolbox 0.4.0 → 0.5.0

data/lib/3scale_toolbox/commands/copy_command.rb

@@ -0,0 +1,20 @@
+require 'cri'
+require '3scale_toolbox/base_command'
+require '3scale_toolbox/commands/copy_command/copy_service'
+
+module ThreeScaleToolbox
+  module Commands
+    module CopyCommand
+      extend ThreeScaleToolbox::Command
+      def self.command
+        Cri::Command.define do
+          name        'copy'
+          usage       'copy <command> [options]'
+          summary     '3scale copy command'
+          description '3scale copy command.'
+        end
+      end
+      add_subcommand(CopyServiceSubcommand)
+    end
+  end
+end

data/lib/3scale_toolbox/commands/help_command.rb

@@ -0,0 +1,13 @@
+require 'cri'
+require '3scale_toolbox/base_command'
+
+module ThreeScaleToolbox
+  module Commands
+    module HelpCommand
+      extend ThreeScaleToolbox::Command
+      def self.command
+        Cri::Command.new_basic_help
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command/import_csv.rb

@@ -0,0 +1,175 @@
+require 'cri'
+require 'uri'
+require 'csv'
+require '3scale/api'
+require '3scale_toolbox/base_command'
+
+module ThreeScaleToolbox
+  module Commands
+    module ImportCommand
+      module ImportCsvSubcommand
+        extend ThreeScaleToolbox::Command
+        def self.command
+          Cri::Command.define do
+            name        'csv'
+            usage       'csv [opts] -d <dst> -f <file>'
+            summary     'Import csv file'
+            description 'Create new services, metrics, methods and mapping rules from CSV formatted file'
+
+            required  :d, :destination, '3scale target instance. Format: "http[s]://<provider_key>@3scale_url"'
+            required  :f, 'file', 'CSV formatted file'
+
+            run do |opts, args, _|
+              ImportCsvSubcommand.run opts, args
+            end
+          end
+        end
+
+        def self.exit_with_message(message)
+          puts message
+          exit 1
+        end
+
+        def self.fetch_required_option(options, key)
+          options.fetch(key) { exit_with_message "error: Missing argument #{key}" }
+        end
+
+        def self.provider_key_from_url(url)
+          URI(url).user
+        end
+
+        def self.endpoint_from_url(url)
+          uri      = URI(url)
+          uri.user = nil
+
+          uri.to_s
+        end
+
+        def self.auth_app_key_according_service(service)
+          case service['backend_version']
+          when '1'
+            'user_key'
+          when '2'
+            'app_id'
+          when 'oauth'
+            'oauth'
+          end
+        end
+
+        def self.import_csv(destination, file_path, insecure)
+          endpoint     = endpoint_from_url destination
+          provider_key = provider_key_from_url destination
+
+          client   = ThreeScale::API.new(endpoint: endpoint,
+                                         provider_key: provider_key,
+                                         verify_ssl: !insecure
+                                        )
+          data     = CSV.read file_path
+          headings = data.shift
+          services = {}
+          stats    = { services: 0, metrics: 0, methods: 0 , mapping_rules: 0 }
+
+          # prepare services data
+          data.each do |row|
+            service_name = row[headings.find_index('service_name')]
+            item         = {}
+
+            services[service_name] ||= {}
+            services[service_name][:items] ||= []
+
+            (headings - ['service_name']).each do |heading|
+              item[heading] = row[headings.find_index(heading)]
+            end
+
+            services[service_name][:items].push item
+          end
+
+          services.keys.each do |service_name|
+            # create service
+            service = client.create_service name: service_name
+
+            if service['errors'].nil?
+              stats[:services] += 1
+              puts "Service #{service_name} has been created."
+            else
+              abort "Service has not been saved. Errors: #{service['errors']}"
+            end
+
+            # find hits metric (default)
+            hits_metric = client.list_metrics(service['id']).find do |metric|
+              metric['system_name'] == 'hits'
+            end
+
+            services[service_name][:items].each do |item|
+
+              metric, method = {}
+
+              case item['type']
+                # create a metric
+              when 'metric'
+                metric = client.create_metric(service['id'], {
+                  system_name:   item['endpoint_system_name'],
+                  friendly_name: item['endpoint_name'],
+                  unit:          'unit'
+                })
+
+                if metric['errors'].nil?
+                  stats[:metrics] += 1
+                  puts "Metric #{item['endpoint_name']} has been created."
+                else
+                  puts "Metric has not been saved. Errors: #{metric['errors']}"
+                end
+                # create a method
+              when 'method'
+                method = client.create_method(service['id'], hits_metric['id'], {
+                  system_name:   item['endpoint_system_name'],
+                  friendly_name: item['endpoint_name'],
+                  unit:          'unit'
+                })
+
+                if method['errors'].nil?
+                  stats[:methods] += 1
+                  puts "Method #{item['endpoint_name']} has been created."
+                else
+                  puts "Method has not been saved. Errors: #{method['errors']}"
+                end
+              end
+
+              # create a mapping rule
+              if (metric_id = metric['id'] || method['id'])
+                mapping_rule = client.create_mapping_rule(service['id'], {
+                  metric_id:          metric_id,
+                  pattern:            item['endpoint_path'],
+                  http_method:        item['endpoint_http_method'],
+                  metric_system_name: item['endpoint_system_name'],
+                  auth_app_key:       auth_app_key_according_service(service),
+                  delta:              1
+                })
+
+                if mapping_rule['errors'].nil?
+                  stats[:mapping_rules] += 1
+                  puts "Mapping rule #{item['endpoint_system_name']} has been created."
+                else
+                  puts "Mapping rule has not been saved. Errors: #{mapping_rule['errors']}"
+                end
+              end
+            end
+          end
+
+          puts "#{services.keys.count} services in CSV file"
+          puts "#{stats[:services]} services have been created"
+          puts "#{stats[:metrics]} metrics have been created"
+          puts "#{stats[:methods]} methods have beeen created"
+          puts "#{stats[:mapping_rules]} mapping rules have been created"
+        end
+
+        def self.run(opts, _)
+          destination = fetch_required_option(opts, :destination)
+          file_path = fetch_required_option(opts, :file)
+          insecure = opts[:insecure] || false
+          import_csv(destination, file_path, insecure)
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command.rb

@@ -0,0 +1,20 @@
+require 'cri'
+require '3scale_toolbox/base_command'
+require '3scale_toolbox/commands/import_command/import_csv'
+
+module ThreeScaleToolbox
+  module Commands
+    module ImportCommand
+      extend ThreeScaleToolbox::Command
+      def self.command
+        Cri::Command.define do
+          name        'import'
+          usage       'import <command> [options]'
+          summary     '3scale import command'
+          description '3scale import command.'
+        end
+      end
+      add_subcommand(ImportCsvSubcommand)
+    end
+  end
+end

data/lib/3scale_toolbox/commands/update_command/update_service.rb

@@ -0,0 +1,247 @@
+require 'cri'
+require '3scale_toolbox/base_command'
+
+module ThreeScaleToolbox
+  module Commands
+    module UpdateCommand
+      module UpdateServiceSubcommand
+        extend ThreeScaleToolbox::Command
+        def self.command
+          Cri::Command.define do
+            name        'service'
+            usage       'service [opts] -s <src> -d <dst> <src_service_id> <dst_service_id>'
+            summary     'Update service'
+            description 'Will update existing service, update proxy settings, metrics, methods, application plans and mapping rules.'
+
+            required  :s, :source, '3scale source instance. Format: "http[s]://<provider_key>@3scale_url"'
+            required  :d, :destination, '3scale target instance. Format: "http[s]://<provider_key>@3scale_url"'
+            flag      :f, :force, 'Overwrites the mapping rules by deleting all rules from target service first'
+            flag      :r, 'rules-only', 'Updates only the mapping rules'
+
+            run do |opts, args, _|
+              UpdateServiceSubcommand.run opts, args
+            end
+          end
+        end
+
+        def self.exit_with_message(message)
+          puts message
+          exit 1
+        end
+
+        def self.fetch_required_option(options, key)
+          options.fetch(key) { exit_with_message "error: Missing argument #{key}" }
+        end
+
+        class ServiceUpdater
+          attr_reader :source_client, :target_client, :source_service_id, :target_service_id
+
+          def initialize (source, source_service_id, destination, target_service_id, insecure)
+            @source_client = ThreeScale::API.new(
+              endpoint:     endpoint_from_url(source),
+              provider_key: provider_key_from_url(source),
+              verify_ssl: !insecure
+            )
+            @target_client = ThreeScale::API.new(
+              endpoint:     endpoint_from_url(destination),
+              provider_key: provider_key_from_url(destination),
+              verify_ssl: !insecure
+            )
+            @source_service_id = source_service_id
+            @target_service_id = target_service_id
+          end
+
+          def compare_hashes(first, second, keys)
+            keys.map{ |key| first.fetch(key) } == keys.map{ |key| second.fetch(key) }
+          end
+
+          def provider_key_from_url(url)
+            URI(url).user
+          end
+
+          def endpoint_from_url(url)
+            uri      = URI(url)
+            uri.user = nil
+
+            uri.to_s
+          end
+
+          def target_service_params(source)
+            source.select { |k, v| Commands.service_valid_params.include?(k) && v }
+          end
+
+          def source_metrics
+            @source_metrics ||= source_client.list_metrics(source_service_id)
+          end
+
+          def metrics_mapping
+            @metrics_mapping ||= target_client.list_metrics(target_service_id).map do |target|
+              metric = source_metrics.find{|metric| metric.fetch('system_name') == target.fetch('system_name') }
+              metric ||= {}
+
+              [metric['id'], target['id']]
+            end.to_h
+          end
+
+          def copy_service_settings
+            source_service = source_client.show_service(source_service_id)
+            puts "updating service settings for service id #{target_service_id}..."
+            target_update_response = target_client.update_service(target_service_id, target_service_params(source_service))
+            raise "Service has not been saved. Errors: #{target_update_response['errors']}" unless target_update_response['errors'].nil?
+          end
+
+          def copy_proxy_settings
+            puts "updating proxy configuration for service id #{target_service_id}..."
+            proxy = source_client.show_proxy(source_service_id)
+            target_client.update_proxy(target_service_id, proxy)
+            puts "updated proxy of #{target_service_id} to match the source #{source_service_id}"
+          end
+
+          def copy_metrics_and_methods
+            target_metrics = target_client.list_metrics(target_service_id)
+
+            source_hits = source_metrics.find{ |metric| metric['system_name'] == 'hits' } or raise 'missing hits metric'
+            target_hits = target_metrics.find{ |metric| metric['system_name'] == 'hits' } or raise 'missing hits metric'
+
+            source_methods = source_client.list_methods(source_service_id, source_hits['id'])
+            target_methods = target_client.list_methods(target_service_id, target_hits['id'])
+
+            puts "source service hits metric #{source_hits['id']} has #{source_methods.size} methods"
+            puts "target service hits metric #{target_hits['id']} has #{target_methods.size} methods"
+
+            missing_methods = source_methods.reject { |source_method|  target_methods.find{|target_method| compare_hashes(source_method, target_method, ['system_name']) } }
+
+            puts "creating #{missing_methods.size} missing methods on target service..."
+            missing_methods.each do |method|
+              target = { friendly_name: method['friendly_name'], system_name: method['system_name'] }
+              target_client.create_method(target_service_id, target_hits['id'], target)
+            end
+
+            target_metrics = target_client.list_metrics(target_service_id)
+
+            puts "source service has #{source_metrics.size} metrics"
+            puts "target service has #{target_metrics.size} metrics"
+
+            missing_metrics = source_metrics.reject { |source_metric| target_metrics.find{|target_metric| compare_hashes(source_metric, target_metric, ['system_name']) } }
+
+            missing_metrics.map do |metric|
+              metric.delete('links')
+              target_client.create_metric(target_service_id, metric)
+            end
+
+            puts "created #{missing_metrics.size} metrics on the target service"
+          end
+
+          def copy_application_plans
+            source_plans = source_client.list_service_application_plans(source_service_id)
+            target_plans = target_client.list_service_application_plans(target_service_id)
+
+            puts "source service has #{source_plans.size} application plans"
+            puts "target service has #{target_plans.size} application plans"
+
+            missing_application_plans = source_plans.reject { |source_plan| target_plans.find{|target_plan| source_plan.fetch('system_name') == target_plan.fetch('system_name') } }
+
+            puts "creating #{missing_application_plans.size} missing application plans..."
+
+            missing_application_plans.each do |plan|
+              plan.delete('links')
+              plan.delete('default') # TODO: handle default plans
+
+              if plan.delete('custom') # TODO: what to do with custom plans?
+                puts "skipping custom plan #{plan}"
+              else
+                target_client.create_application_plan(target_service_id, plan)
+              end
+            end
+
+            puts "updating limits for application plans..."
+
+            application_plan_mapping = target_client.list_service_application_plans(target_service_id).map do |plan_target|
+              plan = source_plans.find{|plan| plan.fetch('system_name') == plan_target.fetch('system_name') }
+              plan ||= {}
+              [plan['id'], plan_target['id']]
+            end.to_h.reject { |key, value| !key }
+
+            application_plan_mapping.each do |source_id, target_id|
+              source_limits = source_client.list_application_plan_limits(source_id)
+              target_limits = target_client.list_application_plan_limits(target_id)
+
+              missing_limits = source_limits.reject { |limit| target_limits.find{|limit_target| limit.fetch('period') == limit_target.fetch('period') } }
+
+              puts "target application plan #{target_id} is missing #{missing_limits.size} from the source plan #{source_id}"
+
+              missing_limits.each do |limit|
+                limit.delete('links')
+                target_client.create_application_plan_limit(target_id, metrics_mapping.fetch(limit.fetch('metric_id')), limit)
+              end
+
+            end
+          end
+
+          def copy_mapping_rules force_mapping_rules
+            source_mapping_rules = source_client.list_mapping_rules(source_service_id)
+            target_mapping_rules = target_client.list_mapping_rules(target_service_id)
+
+            puts "the source service has #{source_mapping_rules.size} mapping rules"
+            puts "the target has #{target_mapping_rules.size} mapping rules"
+
+            if force_mapping_rules
+              puts "force mode was chosen, deleting existing mapping rules on target service..."
+              target_mapping_rules.each do |rule|
+                target_client.delete_mapping_rule(target_service_id, rule['id'])
+              end
+              missing_mapping_rules = source_mapping_rules
+            else
+              unique_target_mapping_rules = target_mapping_rules.dup
+
+              missing_mapping_rules = source_mapping_rules.reject do |mapping_rule|
+                matching_metric = unique_target_mapping_rules.find do |target|
+                  compare_hashes(mapping_rule, target, %w(pattern http_method delta)) &&
+                    metrics_mapping.fetch(mapping_rule.fetch('metric_id')) == target.fetch('metric_id')
+                end
+
+                unique_target_mapping_rules.delete(matching_metric)
+              end
+            end
+
+            puts "missing #{missing_mapping_rules.size} mapping rules"
+
+            missing_mapping_rules.each do |mapping_rule|
+              mapping_rule.delete('links')
+              mapping_rule['metric_id'] = metrics_mapping.fetch(mapping_rule.delete('metric_id'))
+              target_client.create_mapping_rule(target_service_id, mapping_rule)
+            end
+            puts "created #{missing_mapping_rules.size} mapping rules"
+          end
+
+          def update_service force_mapping_rules=false
+            copy_service_settings
+            copy_proxy_settings
+            copy_metrics_and_methods
+            copy_application_plans
+            copy_mapping_rules force_mapping_rules
+          end
+        end
+
+        def self.run(opts, args)
+          source      = fetch_required_option(opts, :source)
+          destination = fetch_required_option(opts, :destination)
+          insecure    = opts[:insecure] || false
+          exit_with_message 'error: missing source_service_id argument' if args.empty?
+          exit_with_message 'error: missing target_service_id argument' if args.size < 2
+          source_service_id = args[0]
+          target_service_id = args[1]
+          force_update = opts[:force] || false
+          rules_only = opts[:'rules-only'] || false
+          updater = ServiceUpdater.new(source, source_service_id, destination, target_service_id, insecure)
+
+          if rules_only
+            updater.copy_mapping_rules force_update
+          else
+            updater.update_service force_update
+          end
+        end
+      end
+    end
+  end
+end