3scale_toolbox 0.16.0 → 0.18.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77ce2f220c416d634541773317d35da8018a75d7e3d91c009338b7a971614eee
-  data.tar.gz: 4d5ee60b4cc55c0ba002fc007ba578801012c9c5580b4fe540f03c3c9dbb2d26
+  metadata.gz: 0a3084c0b2b684a252c7e07c4c9a87e3087ada1ad6a677766aa508ac98e1f313
+  data.tar.gz: 158b1b008a67836c2b27ccd8cbcd49f59995e4ac776eae93c59bf44b002cbf3a
 SHA512:
-  metadata.gz: 947d84f88a8b8e673ea313ba79a3e72eda4b03b85af30f12003b8c6b867f98addf8bc0eebea2e4a3c3bdbc64f80f241346c0e70bdb2ec9e5e9b3a698ab539438
-  data.tar.gz: 106e99be0a64a27284d6b7a64b796ac0ef6b14e30011b3d42366d1e6494e9f623fdb6221924204830e4e9dbb59a6086b0aac8601a4e7b5a968a59f9bfaefed61
+  metadata.gz: '02789c23291a2191617a56c3456297539b09e0922bbf62b57b613065b04d8951800d91f49989d510258c0c3b1baa308d1219f4c29373879c8a65f9a93bd4653a'
+  data.tar.gz: abaca7089c51b5de57c118f5154a74466dd6671a96d7ec043b47ba0d4772761a2d9896b1195e762115ff121296c16cc2f195a5d39e9c20191393f45e6ee756bc

data/3scale_toolbox.gemspec

@@ -31,12 +31,12 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'dotenv'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.8'
   spec.add_development_dependency 'webmock', '~> 3.4'
   spec.required_ruby_version = '>= 2.5'
 
-  spec.add_dependency '3scale-api', '~> 1.0'
+  spec.add_dependency '3scale-api', '~> 1.4'
   spec.add_dependency 'cri', '~> 2.15'
   spec.add_dependency 'json-schema', '~> 2.8'
   spec.add_dependency 'oas_parser', '~> 0.20'

data/README.md

@@ -1,9 +1,7 @@
 # 3scale toolbox
 [![CircleCI](https://circleci.com/gh/3scale/3scale_toolbox.svg?style=svg)](https://circleci.com/gh/3scale/3scale_toolbox)
-
-This software is licensed under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0).
-
-See the LICENSE and NOTICE files that should have been provided along with this software for details.
+[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](http://www.apache.org/licenses/LICENSE-2.0)
+[![GitHub release](https://img.shields.io/github/v/release/3scale/3scale_toolbox.svg)](https://github.com/3scale/3scale_toolbox/releases/latest)
 
 ## Description
 3scale toolbox is a set of tools to help you manage your 3scale product. Using the [3scale API Ruby Client](https://github.com/3scale/3scale-api-ruby).
@@ -15,6 +13,7 @@ See the LICENSE and NOTICE files that should have been provided along with this
    * [Copy service](docs/copy-service.md)
    * [Copy backend](docs/copy-backend.md)
    * [Copy product](docs/copy-product.md)
+   * [Export/Import product](docs/export-import-product.md)
    * [Import from CSV](docs/import-csv.md)
    * [Import from OpenAPI definition](docs/openapi.md)
    * [Export/Import Application Plan](docs/export-import-app-plan.md)
@@ -23,15 +22,17 @@ See the LICENSE and NOTICE files that should have been provided along with this
    * Create, Apply, List, Delete [Method](docs/method.md)
    * Create, Apply, List, Show, Delete [Service](docs/service.md)
    * Create, Apply, List, Delete [ActiveDocs](docs/activedocs.md)
-   * List, Show, Promote [Proxy Configuration](docs/proxy-config.md)
+   * List, Show, Promote, Export, Deploy [Proxy Configuration](docs/proxy-config.md)
    * [Copy Policy Registry](docs/copy-policy-registry.md)
    * Create, Apply, List, Show, Delete, Suspend, Resume [Applications](docs/applications.md)
+   * [Export/Import Product Policy Chain](docs/export-import-policy-chain.md)
    * [Remotes](docs/remotes.md)
 * [Development](#development)
    * [Testing](#testing)
    * [Develop your own core command](#develop-core-command)
    * [Licenses](#licenses)
 * [Plugins](#plugins)
+* [Error Reporting](docs/errors.md)
 * [Troubleshooting](#troubleshooting)
 * [Contributing](#contributing)
 
@@ -40,6 +41,7 @@ Supported Ruby interpreters
 
 * MRI 2.5
 * MRI 2.6
+* MRI 2.7
 
 ## Installation
 Install the toolbox:
@@ -82,10 +84,11 @@ COMMANDS
     method               method super command
     metric               metric super command
     policy-registry      policy-registry super command
+    product              product super command
     proxy-config         proxy-config super command
     remote               remotes super command
     service              services super command
-    update               [DEPRECTATED] update super command
+    update               [DEPRECATED] update super command
 
 OPTIONS
     -c --config-file=<value>      3scale toolbox configuration file (default:
@@ -120,7 +123,7 @@ rake spec:integration
 
 Integration tests can be run locally or against a real 3scale account.
 When details of the account are set via environment variables,
-integration tests are run agains given account.
+integration tests are run against the given account.
 Otherwise, tests are run locally with mocked 3scale clients.
 
 The easiest way to set everything up is it to have a `.env` file in the root of the project with the following environment variables (set your own values):
@@ -163,7 +166,7 @@ end
 A few things worth highlighting:
 - Your module must include the *ThreeScaleToolbox::Command* module. It allows your command to be added to the toobox command tree.
 - You must implement the `command` module function and return an instance of `Cri::Command` from [cri](https://github.com/ddfreyne/cri)
-- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from remote list and client instantiation is done out of the box.
+- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from the remote list and client instantiation is done out of the box.
 
 Then register the core command in `lib/3scale_toolbox/commands.rb`
 ```

data/lib/3scale_toolbox.rb

@@ -10,15 +10,18 @@ require 'oas_parser'
 require '3scale/api'
 require 'json-schema'
 require 'erb'
+require 'logger'
 
 require '3scale_toolbox/version'
 require '3scale_toolbox/helper'
 require '3scale_toolbox/error'
+require '3scale_toolbox/remote_cache'
 require '3scale_toolbox/proxy_logger'
 require '3scale_toolbox/resource_reader'
 require '3scale_toolbox/configuration'
 require '3scale_toolbox/remotes'
 require '3scale_toolbox/3scale_client_factory'
+require '3scale_toolbox/crds'
 require '3scale_toolbox/entities'
 require '3scale_toolbox/attribute_filters'
 require '3scale_toolbox/base_command'

data/lib/3scale_toolbox/3scale_client_factory.rb

@@ -22,10 +22,9 @@ module ThreeScaleToolbox
         remote = remotes.fetch(remote_str)
       end
 
-      client = remote_client(remote.merge(verify_ssl: verify_ssl))
-      return ProxyLogger.new(client) if verbose
-
-      client
+      client = remote_client(**remote.merge(verify_ssl: verify_ssl))
+      client = ProxyLogger.new(client) if verbose
+      RemoteCache.new(client)
     end
 
     private

data/lib/3scale_toolbox/cli/error_handler.rb

@@ -20,10 +20,9 @@ module ThreeScaleToolbox
 
       def handle_error(error)
         if expected_error?(error)
-          warn
-          warn "\e[1m\e[31mError: #{error.message}\e[0m"
+          warn error_serialize(error)
         else
-          print_error(error)
+          print_unexpected_error(error)
         end
       end
 
@@ -36,21 +35,12 @@ module ThreeScaleToolbox
         end
       end
 
-      def print_error(error)
-        write_error(error, $stderr)
-
+      def print_unexpected_error(error)
         File.open('crash.log', 'w') do |io|
           write_verbose_error(error, io)
         end
 
-        write_section_header($stderr, 'Detailed information')
-        warn
-        warn 'A detailed crash log has been written to ./crash.log.'
-      end
-
-      def write_error(error, stream)
-        write_error_message(error, stream)
-        write_stack_trace(error, stream)
+        warn error_serialize(UnexpectedError.new(error))
       end
 
       def write_error_message(error, stream)
@@ -113,6 +103,19 @@ module ThreeScaleToolbox
         stream.puts "===== #{title.upcase}:"
         stream.puts
       end
+
+      def error_serialize(error)
+        JSON.pretty_generate format_error(error)
+      end
+
+      def format_error(error)
+        {
+          code: error.code,
+          message: error.message,
+          class: error.kind,
+          stacktrace: error.stacktrace
+        }.compact
+      end
     end
   end
 end

data/lib/3scale_toolbox/commands.rb

@@ -15,6 +15,7 @@ require '3scale_toolbox/commands/policy_registry_command'
 require '3scale_toolbox/commands/application_command'
 require '3scale_toolbox/commands/backend_command'
 require '3scale_toolbox/commands/product_command'
+require '3scale_toolbox/commands/policies_command'
 
 module ThreeScaleToolbox
   module Commands
@@ -35,6 +36,7 @@ module ThreeScaleToolbox
       ThreeScaleToolbox::Commands::ApplicationCommand,
       ThreeScaleToolbox::Commands::BackendCommand,
       ThreeScaleToolbox::Commands::ProductCommand,
+      ThreeScaleToolbox::Commands::PoliciesCommand,
     ].freeze
   end
 end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_mapping_rules_task.rb

@@ -7,42 +7,26 @@ module ThreeScaleToolbox
 
           # entrypoint
           def run
-            missing_rules = missing_mapping_rules(source_backend.mapping_rules,
-                                                  target_backend.mapping_rules, metrics_map)
             missing_rules.each do |mapping_rule|
-              mapping_rule.metric_id = metrics_map.fetch(mapping_rule.metric_id)
-              Entities::BackendMappingRule.create(backend: target_backend,
-                                                  attrs: mapping_rule.attrs)
+              mr_attrs = mapping_rule.attrs.merge('metric_id' => metrics_map.fetch(mapping_rule.metric_id))
+              Entities::BackendMappingRule.create(backend: target_backend, attrs: mr_attrs)
             end
-            puts "created #{missing_rules.size} mapping rules"
+            logger.info "created #{missing_rules.size} mapping rules"
+            report['missing_mapping_rules_created'] = missing_rules.size
           end
 
           private
 
           def metrics_map
-            @metrics_map ||= build_metrics_mapping
+            @metrics_map ||= source_backend.metrics_mapping(target_backend)
           end
 
-          def build_metrics_mapping
-            target_mm = target_metrics + target_methods
-            source_mm = source_metrics + source_methods
-            target_mm.map do |target|
-              source = source_mm.find do |m|
-                m.system_name == target.system_name
-              end
-              next if source.nil?
-
-              [source.id, target.id]
-            end.compact.to_h
-          end
-
-          def missing_mapping_rules(source_rules, target_rules, metrics_map)
-            ThreeScaleToolbox::Helper.array_difference(source_rules, target_rules) do |source, target|
-              # map metric_id to the target backend domain
-              source_attrs = source.attrs.merge('metric_id' => metrics_map.fetch(source.metric_id))
-              ThreeScaleToolbox::Helper.compare_hashes(source_attrs,
-                                                       target.attrs,
-                                                       %w[pattern http_method delta metric_id])
+          def missing_rules
+            @missing_rules ||= ThreeScaleToolbox::Helper.array_difference(source_backend.mapping_rules, target_backend.mapping_rules) do |source_rule, target_rule|
+              source_rule.pattern == target_rule.pattern &&
+                source_rule.http_method == target_rule.http_method &&
+                source_rule.delta == target_rule.delta &&
+                metrics_map.fetch(source_rule.metric_id) == target_rule.metric_id
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_methods_task.rb

@@ -8,28 +8,23 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_methods.each(&method(:create_method))
-            puts "created #{missing_methods.size} missing methods"
-            invalidate_target_methods if missing_methods.size.positive?
+            logger.info "created #{missing_methods.size} missing methods"
+            report['missing_methods_created'] = missing_methods.size
           end
 
           private
 
           def create_method(method)
-            # return silently if target metric hits does not exist
-            return if target_hits.nil?
-
-            Entities::BackendMethod.create(backend: target_backend,
-                                           parent_id: target_hits.id,
-                                           attrs:  method.attrs)
+            Entities::BackendMethod.create(backend: target_backend, attrs:  method.attrs)
           rescue ThreeScaleToolbox::ThreeScaleApiError => e
             raise e unless ThreeScaleToolbox::Helper.system_name_already_taken_error?(e.apierrors)
 
-            warn "[WARN] backend method #{method.attrs.fetch('system_name')} not created. " \
+            warn "[WARN] backend method #{method.system_name} not created. " \
               'Backend metric with the same system_name exists.'
           end
 
           def missing_methods
-            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_methods, target_methods) do |source, target|
+            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_backend.methods, target_backend.methods) do |source, target|
               source.system_name == target.system_name
             end
           end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_metrics_task.rb

@@ -8,8 +8,8 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_metrics.each(&method(:create_metric))
-            puts "created #{missing_metrics.size} missing metrics"
-            invalidate_target_metrics if missing_metrics.size.positive?
+            logger.info "created #{missing_metrics.size} missing metrics"
+            report['missing_metrics_created'] = missing_metrics.size
           end
 
           private
@@ -19,8 +19,8 @@ module ThreeScaleToolbox
           end
 
           def missing_metrics
-            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_metrics, target_metrics) do |source, target|
-              source.system_name == target.system_name
+            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_backend.metrics, target_backend.metrics) do |s_m, t_m|
+              s_m.system_name == t_m.system_name
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/create_or_update_target_backend_task.rb

@@ -23,8 +23,9 @@ module ThreeScaleToolbox
             # assign target backend for other tasks to have it available
             self.target_backend = backend
 
-            puts "source backend ID: #{source_backend.id} system_name: #{source_backend.attrs['system_name']}"
-            puts "target backend ID: #{target_backend.id} system_name: #{target_backend.attrs['system_name']}"
+            logger.info "source backend ID: #{source_backend.id} system_name: #{source_backend.system_name}"
+            logger.info "target backend ID: #{target_backend.id} system_name: #{target_backend.system_name}"
+            report['backend_id'] = target_backend.id
           end
 
           def create_attrs

data/lib/3scale_toolbox/commands/backend_command/copy_command/task.rb

@@ -26,38 +26,6 @@ module ThreeScaleToolbox
             context[:source_backend] ||= find_source_backend
           end
 
-          def source_methods
-            context[:source_methods] ||= source_backend.methods(source_hits)
-          end
-
-          def source_hits
-            context[:source_hits] ||= source_backend.hits
-          end
-
-          def source_metrics
-            context[:source_metrics] ||= source_backend.metrics
-          end
-
-          def target_metrics
-            context[:target_metrics] ||= target_backend.metrics
-          end
-
-          def target_hits
-            context[:target_hits] ||= target_backend.hits
-          end
-
-          def target_methods
-            context[:target_methods] ||= target_backend.methods(target_hits)
-          end
-
-          def invalidate_target_methods
-            context[:target_methods] = nil
-          end
-
-          def invalidate_target_metrics
-            context[:target_metrics] = nil
-          end
-
           def source_remote
             context[:source_remote]
           end
@@ -75,6 +43,16 @@ module ThreeScaleToolbox
             context[:option_target_system_name]
           end
 
+          def logger
+            context[:logger] ||= Logger.new($stdout).tap do |logger|
+              logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+            end
+          end
+
+          def report
+            context[:report] ||= {}
+          end
+
           private
 
           def find_source_backend

data/lib/3scale_toolbox/commands/import_command/issuer_type_transformer.rb

@@ -0,0 +1,16 @@
+module ThreeScaleToolbox
+  module Commands
+    module ImportCommand
+      module OpenAPI
+        # Helper class to validate values for the oidc-issuer-type argument of the import openapi command
+        class IssuerTypeTransformer
+          def call(issuer_type)
+            raise unless %w[rest keycloak].include?(issuer_type)
+
+            issuer_type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command/openapi.rb

@@ -9,6 +9,7 @@ require '3scale_toolbox/commands/import_command/openapi/create_activedocs_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_proxy_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_oidc_conf_step'
 require '3scale_toolbox/commands/import_command/openapi/update_policies_step'
+require '3scale_toolbox/commands/import_command/issuer_type_transformer'
 
 module ThreeScaleToolbox
   module Commands
@@ -30,6 +31,7 @@ module ThreeScaleToolbox
               flag    nil, 'activedocs-hidden', 'Create ActiveDocs in hidden state'
               flag    nil, 'skip-openapi-validation', 'Skip OpenAPI schema validation'
               flag    nil, 'prefix-matching', 'Use prefix matching instead of strict matching on mapping rules derived from openapi operations'
+              option  nil, 'oidc-issuer-type', 'OIDC Issuer Type (rest, keycloak)', argument: :required, transform: IssuerTypeTransformer.new            
               option  nil, 'oidc-issuer-endpoint', 'OIDC Issuer Endpoint', argument: :required
               option  nil, 'default-credentials-userkey', 'Default credentials policy userkey', argument: :required
               option  nil, 'override-private-basepath', 'Override the basepath for the private URLs', argument: :required
@@ -77,6 +79,7 @@ module ThreeScaleToolbox
               threescale_client: threescale_client(fetch_required_option(:destination)),
               target_system_name: options[:target_system_name],
               activedocs_published: !options[:'activedocs-hidden'],
+              oidc_issuer_type: options[:'oidc-issuer-type'],
               oidc_issuer_endpoint: options[:'oidc-issuer-endpoint'],
               default_credentials_userkey: options[:'default-credentials-userkey'],
               skip_openapi_validation: options[:'skip-openapi-validation'],

data/lib/3scale_toolbox/commands/import_command/openapi/create_activedocs_step.rb

@@ -54,8 +54,9 @@ module ThreeScaleToolbox
             Helper.hash_deep_dup(resource).tap do |activedocs|
               # public production base URL
               # the basePath field is updated to a new value only when overriden by optional param
-              api_spec.set_server_url(activedocs,
-                                      URI.join(service.proxy.fetch('endpoint'), public_base_path))
+              unless service.proxy['endpoint'].nil?
+                api_spec.set_server_url(activedocs, URI.join(service.proxy.fetch('endpoint'), public_base_path))
+              end
               # security definitions
               # just valid for oauth2 when oidc_issuer_endpoint is supplied
               if !api_spec.security.nil? && api_spec.security[:type] == 'oauth2' && !oidc_issuer_endpoint.nil?