3scale_toolbox 0.17.1 → 0.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2757cb4872a0b79aecc940e202197af6474d8d4057b2b576b1e094315a3c91c6
-  data.tar.gz: 84702d1a9d6430db62c85689ae4c18f2a1cab152ffd3dc2ea4ac8b8b6c22df39
+  metadata.gz: aaf0f178f318385464342bd932cc8e0e78f3e8a291e2226204692d62c1d2dca7
+  data.tar.gz: a5aeca4f1da609adb33fc11a5cecd1ec435fcdef8c08c8755796da768c47e9ea
 SHA512:
-  metadata.gz: 6e19f6db5644ecdd2825aba35bc1113dbfb45a19ee66ed8caa3f1a1295febaf158a2d39b66d1755c8f4b2826e2334fe149f3711c7c75d6bdfd31544fee5f5d31
-  data.tar.gz: 838988ab2096976ec1bda3b94a0cea0e5bcefbca49ecf8cbe56a6fee0648a8e4ed2dba26819da3e061ceae9bdba8a1191899ead70b8fe0eb8cf73e9d29a2ef3a
+  metadata.gz: 8521f2b23f6724ea4d9b62064710d22cd884cb2d518df211792b764cc7faaf753644801da6576d9fd61ce872b1503314346e842267632f8ff4b7b7a266658bc3
+  data.tar.gz: '099dcc4097d3d16803c30b8834e9db9985a3fc46b393f2bbfa31cb9eb5f646c5d9b729ece1fd63a7771b180d1a6d03246716163d2a42935a2b9cf2cec00a5817'

data/3scale_toolbox.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |spec|
   spec.name          = '3scale_toolbox'
   spec.version       = ThreeScaleToolbox::VERSION
   spec.licenses      = ['MIT']
-  spec.authors       = ['Michal Cichra', 'Eguzki Astiz Lezaun']
-  spec.email         = ['michal@3scale.net', 'eastizle@redhat.com']
+  spec.authors       = ['Miguel Soriano', 'Eguzki Astiz Lezaun']
+  spec.email         = ['msoriano@redhat.com', 'eastizle@redhat.com']
 
   spec.summary       = %q{3scale Toolbox.}
   spec.description   = %q{3scale tools to manage your API from the terminal.}
@@ -34,9 +34,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.8'
   spec.add_development_dependency 'webmock', '~> 3.4'
-  spec.required_ruby_version = '>= 2.5'
+  spec.required_ruby_version = '>= 2.6'
 
-  spec.add_dependency '3scale-api', '~> 1.1'
+  spec.add_dependency '3scale-api', '~> 1.4'
   spec.add_dependency 'cri', '~> 2.15'
   spec.add_dependency 'json-schema', '~> 2.8'
   spec.add_dependency 'oas_parser', '~> 0.20'

data/README.md

@@ -13,6 +13,7 @@
    * [Copy service](docs/copy-service.md)
    * [Copy backend](docs/copy-backend.md)
    * [Copy product](docs/copy-product.md)
+   * [Export/Import product](docs/export-import-product.md)
    * [Import from CSV](docs/import-csv.md)
    * [Import from OpenAPI definition](docs/openapi.md)
    * [Export/Import Application Plan](docs/export-import-app-plan.md)
@@ -21,9 +22,10 @@
    * Create, Apply, List, Delete [Method](docs/method.md)
    * Create, Apply, List, Show, Delete [Service](docs/service.md)
    * Create, Apply, List, Delete [ActiveDocs](docs/activedocs.md)
-   * List, Show, Promote, Export [Proxy Configuration](docs/proxy-config.md)
+   * List, Show, Promote, Export, Deploy [Proxy Configuration](docs/proxy-config.md)
    * [Copy Policy Registry](docs/copy-policy-registry.md)
    * Create, Apply, List, Show, Delete, Suspend, Resume [Applications](docs/applications.md)
+   * [Export/Import Product Policy Chain](docs/export-import-policy-chain.md)
    * [Remotes](docs/remotes.md)
 * [Development](#development)
    * [Testing](#testing)
@@ -37,8 +39,8 @@
 ## Requirements
 Supported Ruby interpreters
 
-* MRI 2.5
 * MRI 2.6
+* MRI 2.7
 
 ## Installation
 Install the toolbox:
@@ -120,7 +122,7 @@ rake spec:integration
 
 Integration tests can be run locally or against a real 3scale account.
 When details of the account are set via environment variables,
-integration tests are run agains given account.
+integration tests are run against the given account.
 Otherwise, tests are run locally with mocked 3scale clients.
 
 The easiest way to set everything up is it to have a `.env` file in the root of the project with the following environment variables (set your own values):
@@ -163,7 +165,7 @@ end
 A few things worth highlighting:
 - Your module must include the *ThreeScaleToolbox::Command* module. It allows your command to be added to the toobox command tree.
 - You must implement the `command` module function and return an instance of `Cri::Command` from [cri](https://github.com/ddfreyne/cri)
-- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from remote list and client instantiation is done out of the box.
+- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from the remote list and client instantiation is done out of the box.
 
 Then register the core command in `lib/3scale_toolbox/commands.rb`
 ```
@@ -216,5 +218,5 @@ Install, uninstall and update plugins using tools like [RubyGems](https://guides
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/3scale/3scale_toolbox.
+If you are interested in contributing to 3scale Toolbox, please refer to instructions available [here](docs/contributing.md)
 

data/lib/3scale_toolbox.rb

@@ -10,15 +10,18 @@ require 'oas_parser'
 require '3scale/api'
 require 'json-schema'
 require 'erb'
+require 'logger'
 
 require '3scale_toolbox/version'
 require '3scale_toolbox/helper'
 require '3scale_toolbox/error'
+require '3scale_toolbox/remote_cache'
 require '3scale_toolbox/proxy_logger'
 require '3scale_toolbox/resource_reader'
 require '3scale_toolbox/configuration'
 require '3scale_toolbox/remotes'
 require '3scale_toolbox/3scale_client_factory'
+require '3scale_toolbox/crds'
 require '3scale_toolbox/entities'
 require '3scale_toolbox/attribute_filters'
 require '3scale_toolbox/base_command'

data/lib/3scale_toolbox/3scale_client_factory.rb

@@ -22,10 +22,9 @@ module ThreeScaleToolbox
         remote = remotes.fetch(remote_str)
       end
 
-      client = remote_client(remote.merge(verify_ssl: verify_ssl))
-      return ProxyLogger.new(client) if verbose
-
-      client
+      client = remote_client(**remote.merge(verify_ssl: verify_ssl))
+      client = ProxyLogger.new(client) if verbose
+      RemoteCache.new(client)
     end
 
     private

data/lib/3scale_toolbox/commands.rb

@@ -15,6 +15,7 @@ require '3scale_toolbox/commands/policy_registry_command'
 require '3scale_toolbox/commands/application_command'
 require '3scale_toolbox/commands/backend_command'
 require '3scale_toolbox/commands/product_command'
+require '3scale_toolbox/commands/policies_command'
 
 module ThreeScaleToolbox
   module Commands
@@ -35,6 +36,7 @@ module ThreeScaleToolbox
       ThreeScaleToolbox::Commands::ApplicationCommand,
       ThreeScaleToolbox::Commands::BackendCommand,
       ThreeScaleToolbox::Commands::ProductCommand,
+      ThreeScaleToolbox::Commands::PoliciesCommand,
     ].freeze
   end
 end

data/lib/3scale_toolbox/commands/backend_command/copy_command.rb

@@ -2,6 +2,7 @@ require '3scale_toolbox/commands/backend_command/copy_command/task'
 require '3scale_toolbox/commands/backend_command/copy_command/create_or_update_target_backend_task'
 require '3scale_toolbox/commands/backend_command/copy_command/copy_metrics_task'
 require '3scale_toolbox/commands/backend_command/copy_command/copy_methods_task'
+require '3scale_toolbox/commands/backend_command/copy_command/delete_mapping_rules_task'
 require '3scale_toolbox/commands/backend_command/copy_command/copy_mapping_rules_task'
 
 module ThreeScaleToolbox
@@ -41,6 +42,7 @@ module ThreeScaleToolbox
           # First metrics as methods need 'hits' metric in target backend
           tasks << CopyCommand::CopyMetricsTask.new(context)
           tasks << CopyCommand::CopyMethodsTask.new(context)
+          tasks << CopyCommand::DeleteMappingRulesTask.new(context)
           tasks << CopyCommand::CopyMappingRulesTask.new(context)
           tasks.each(&:call)
         end
@@ -56,6 +58,7 @@ module ThreeScaleToolbox
             source_remote: threescale_client(fetch_required_option(:source)),
             target_remote: threescale_client(fetch_required_option(:destination)),
             source_backend_ref: arguments[:source_backend],
+            delete_mapping_rules: true,
             option_target_system_name: options[:'target-system-name']
           }
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_mapping_rules_task.rb

@@ -7,42 +7,26 @@ module ThreeScaleToolbox
 
           # entrypoint
           def run
-            missing_rules = missing_mapping_rules(source_backend.mapping_rules,
-                                                  target_backend.mapping_rules, metrics_map)
             missing_rules.each do |mapping_rule|
-              mapping_rule.metric_id = metrics_map.fetch(mapping_rule.metric_id)
-              Entities::BackendMappingRule.create(backend: target_backend,
-                                                  attrs: mapping_rule.attrs)
+              mr_attrs = mapping_rule.attrs.merge('metric_id' => metrics_map.fetch(mapping_rule.metric_id))
+              Entities::BackendMappingRule.create(backend: target_backend, attrs: mr_attrs)
             end
-            puts "created #{missing_rules.size} mapping rules"
+            logger.info "created #{missing_rules.size} mapping rules"
+            report['missing_mapping_rules_created'] = missing_rules.size
           end
 
           private
 
           def metrics_map
-            @metrics_map ||= build_metrics_mapping
+            @metrics_map ||= source_backend.metrics_mapping(target_backend)
           end
 
-          def build_metrics_mapping
-            target_mm = target_metrics + target_methods
-            source_mm = source_metrics + source_methods
-            target_mm.map do |target|
-              source = source_mm.find do |m|
-                m.system_name == target.system_name
-              end
-              next if source.nil?
-
-              [source.id, target.id]
-            end.compact.to_h
-          end
-
-          def missing_mapping_rules(source_rules, target_rules, metrics_map)
-            ThreeScaleToolbox::Helper.array_difference(source_rules, target_rules) do |source, target|
-              # map metric_id to the target backend domain
-              source_attrs = source.attrs.merge('metric_id' => metrics_map.fetch(source.metric_id))
-              ThreeScaleToolbox::Helper.compare_hashes(source_attrs,
-                                                       target.attrs,
-                                                       %w[pattern http_method delta metric_id])
+          def missing_rules
+            @missing_rules ||= ThreeScaleToolbox::Helper.array_difference(source_backend.mapping_rules, target_backend.mapping_rules) do |source_rule, target_rule|
+              source_rule.pattern == target_rule.pattern &&
+                source_rule.http_method == target_rule.http_method &&
+                source_rule.delta == target_rule.delta &&
+                metrics_map.fetch(source_rule.metric_id) == target_rule.metric_id
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_methods_task.rb

@@ -8,28 +8,23 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_methods.each(&method(:create_method))
-            puts "created #{missing_methods.size} missing methods"
-            invalidate_target_methods if missing_methods.size.positive?
+            logger.info "created #{missing_methods.size} missing methods"
+            report['missing_methods_created'] = missing_methods.size
           end
 
           private
 
           def create_method(method)
-            # return silently if target metric hits does not exist
-            return if target_hits.nil?
-
-            Entities::BackendMethod.create(backend: target_backend,
-                                           parent_id: target_hits.id,
-                                           attrs:  method.attrs)
+            Entities::BackendMethod.create(backend: target_backend, attrs:  method.attrs)
           rescue ThreeScaleToolbox::ThreeScaleApiError => e
             raise e unless ThreeScaleToolbox::Helper.system_name_already_taken_error?(e.apierrors)
 
-            warn "[WARN] backend method #{method.attrs.fetch('system_name')} not created. " \
+            warn "[WARN] backend method #{method.system_name} not created. " \
               'Backend metric with the same system_name exists.'
           end
 
           def missing_methods
-            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_methods, target_methods) do |source, target|
+            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_backend.methods, target_backend.methods) do |source, target|
               source.system_name == target.system_name
             end
           end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_metrics_task.rb

@@ -8,8 +8,8 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_metrics.each(&method(:create_metric))
-            puts "created #{missing_metrics.size} missing metrics"
-            invalidate_target_metrics if missing_metrics.size.positive?
+            logger.info "created #{missing_metrics.size} missing metrics"
+            report['missing_metrics_created'] = missing_metrics.size
           end
 
           private
@@ -19,8 +19,8 @@ module ThreeScaleToolbox
           end
 
           def missing_metrics
-            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_metrics, target_metrics) do |source, target|
-              source.system_name == target.system_name
+            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_backend.metrics, target_backend.metrics) do |s_m, t_m|
+              s_m.system_name == t_m.system_name
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/create_or_update_target_backend_task.rb

@@ -23,8 +23,9 @@ module ThreeScaleToolbox
             # assign target backend for other tasks to have it available
             self.target_backend = backend
 
-            puts "source backend ID: #{source_backend.id} system_name: #{source_backend.attrs['system_name']}"
-            puts "target backend ID: #{target_backend.id} system_name: #{target_backend.attrs['system_name']}"
+            logger.info "source backend ID: #{source_backend.id} system_name: #{source_backend.system_name}"
+            logger.info "target backend ID: #{target_backend.id} system_name: #{target_backend.system_name}"
+            report['backend_id'] = target_backend.id
           end
 
           def create_attrs

data/lib/3scale_toolbox/commands/backend_command/copy_command/delete_mapping_rules_task.rb

@@ -0,0 +1,18 @@
+module ThreeScaleToolbox
+  module Commands
+    module BackendCommand
+      module CopyCommand
+        class DeleteMappingRulesTask
+          include Task
+
+          # entrypoint
+          def run
+            return unless delete_mapping_rules
+
+            target_backend.mapping_rules.each(&:delete)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/backend_command/copy_command/task.rb

@@ -26,38 +26,6 @@ module ThreeScaleToolbox
             context[:source_backend] ||= find_source_backend
           end
 
-          def source_methods
-            context[:source_methods] ||= source_backend.methods(source_hits)
-          end
-
-          def source_hits
-            context[:source_hits] ||= source_backend.hits
-          end
-
-          def source_metrics
-            context[:source_metrics] ||= source_backend.metrics
-          end
-
-          def target_metrics
-            context[:target_metrics] ||= target_backend.metrics
-          end
-
-          def target_hits
-            context[:target_hits] ||= target_backend.hits
-          end
-
-          def target_methods
-            context[:target_methods] ||= target_backend.methods(target_hits)
-          end
-
-          def invalidate_target_methods
-            context[:target_methods] = nil
-          end
-
-          def invalidate_target_metrics
-            context[:target_metrics] = nil
-          end
-
           def source_remote
             context[:source_remote]
           end
@@ -75,6 +43,20 @@ module ThreeScaleToolbox
             context[:option_target_system_name]
           end
 
+          def logger
+            context[:logger] ||= Logger.new($stdout).tap do |logger|
+              logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+            end
+          end
+
+          def report
+            context[:report] ||= {}
+          end
+
+          def delete_mapping_rules
+            context.fetch(:delete_mapping_rules, false)
+          end
+
           private
 
           def find_source_backend

data/lib/3scale_toolbox/commands/import_command/issuer_type_transformer.rb

@@ -0,0 +1,16 @@
+module ThreeScaleToolbox
+  module Commands
+    module ImportCommand
+      module OpenAPI
+        # Helper class to validate values for the oidc-issuer-type argument of the import openapi command
+        class IssuerTypeTransformer
+          def call(issuer_type)
+            raise unless %w[rest keycloak].include?(issuer_type)
+
+            issuer_type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command/openapi.rb

@@ -9,6 +9,7 @@ require '3scale_toolbox/commands/import_command/openapi/create_activedocs_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_proxy_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_oidc_conf_step'
 require '3scale_toolbox/commands/import_command/openapi/update_policies_step'
+require '3scale_toolbox/commands/import_command/issuer_type_transformer'
 
 module ThreeScaleToolbox
   module Commands
@@ -30,6 +31,7 @@ module ThreeScaleToolbox
               flag    nil, 'activedocs-hidden', 'Create ActiveDocs in hidden state'
               flag    nil, 'skip-openapi-validation', 'Skip OpenAPI schema validation'
               flag    nil, 'prefix-matching', 'Use prefix matching instead of strict matching on mapping rules derived from openapi operations'
+              option  nil, 'oidc-issuer-type', 'OIDC Issuer Type (rest, keycloak)', argument: :required, transform: IssuerTypeTransformer.new            
               option  nil, 'oidc-issuer-endpoint', 'OIDC Issuer Endpoint', argument: :required
               option  nil, 'default-credentials-userkey', 'Default credentials policy userkey', argument: :required
               option  nil, 'override-private-basepath', 'Override the basepath for the private URLs', argument: :required
@@ -77,6 +79,7 @@ module ThreeScaleToolbox
               threescale_client: threescale_client(fetch_required_option(:destination)),
               target_system_name: options[:target_system_name],
               activedocs_published: !options[:'activedocs-hidden'],
+              oidc_issuer_type: options[:'oidc-issuer-type'],
               oidc_issuer_endpoint: options[:'oidc-issuer-endpoint'],
               default_credentials_userkey: options[:'default-credentials-userkey'],
               skip_openapi_validation: options[:'skip-openapi-validation'],
@@ -105,6 +108,8 @@ module ThreeScaleToolbox
           end
 
           def openapi_parser
+            raise ThreeScaleToolbox::Error, 'only JSON/YAML format is supported' unless openapi_resource.is_a?(Hash)
+
             if openapi_resource.key?('openapi')
               ThreeScaleToolbox::OpenAPI::OAS3.build(openapi_path, openapi_resource, validate: validate)
             else

data/lib/3scale_toolbox/commands/import_command/openapi/create_activedocs_step.rb

@@ -54,8 +54,9 @@ module ThreeScaleToolbox
             Helper.hash_deep_dup(resource).tap do |activedocs|
               # public production base URL
               # the basePath field is updated to a new value only when overriden by optional param
-              api_spec.set_server_url(activedocs,
-                                      URI.join(service.proxy.fetch('endpoint'), public_base_path))
+              unless service.proxy['endpoint'].nil?
+                api_spec.set_server_url(activedocs, URI.join(service.proxy.fetch('endpoint'), public_base_path))
+              end
               # security definitions
               # just valid for oauth2 when oidc_issuer_endpoint is supplied
               if !api_spec.security.nil? && api_spec.security[:type] == 'oauth2' && !oidc_issuer_endpoint.nil?

data/lib/3scale_toolbox/commands/import_command/openapi/create_mapping_rule_step.rb

@@ -7,7 +7,8 @@ module ThreeScaleToolbox
 
           def call
             operations.each do |op|
-              service.create_mapping_rule(op.mapping_rule)
+              Entities::MappingRule.create(service: service,
+                                           attrs: op.mapping_rule)
               puts "Created #{op.http_method} #{op.pattern} endpoint"
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_method_step.rb

@@ -7,18 +7,13 @@ module ThreeScaleToolbox
 
           def call
             missing_operations.each do |op|
-              method = Entities::Method.create(service: service, parent_id: hits_metric_id,
-                                               attrs: op.method)
+              method = Entities::Method.create(service: service, attrs: op.method)
               op.set(:metric_id, method.id)
             end
 
             existing_operations.each do |op|
-              method_attrs = methods_index.fetch(op.method['system_name'])
-              method = Entities::Method.new(
-                id: method_attrs.fetch('id'),
-                parent_id: hits_metric_id,
-                service: service
-              )
+              method_attrs = methods_index.fetch(op.method['system_name']).attrs
+              method = Entities::Method.new(id: method_attrs.fetch('id'), service: service)
               method.update(op.method)
               op.set(:metric_id, method.id)
             end
@@ -26,13 +21,9 @@ module ThreeScaleToolbox
 
           private
 
-          def hits_metric_id
-            @hits_metric_id ||= service.hits['id']
-          end
-
           def methods_index
-            @methods_index ||= service.methods(hits_metric_id).each_with_object({}) do |method, acc|
-              acc[method['system_name']] = method
+            @methods_index ||= service.methods.each_with_object({}) do |method, acc|
+              acc[method.system_name] = method
             end
           end