3scale_toolbox 0.17.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2757cb4872a0b79aecc940e202197af6474d8d4057b2b576b1e094315a3c91c6
-  data.tar.gz: 84702d1a9d6430db62c85689ae4c18f2a1cab152ffd3dc2ea4ac8b8b6c22df39
+  metadata.gz: d8b53876bf5759606f9517c2c62a32d66587a28ee6425afd1ba011f20222359c
+  data.tar.gz: 35a2783fdf7d5ee796014cbb7d1e3f85b08987b134b168a84ee40c42308b6150
 SHA512:
-  metadata.gz: 6e19f6db5644ecdd2825aba35bc1113dbfb45a19ee66ed8caa3f1a1295febaf158a2d39b66d1755c8f4b2826e2334fe149f3711c7c75d6bdfd31544fee5f5d31
-  data.tar.gz: 838988ab2096976ec1bda3b94a0cea0e5bcefbca49ecf8cbe56a6fee0648a8e4ed2dba26819da3e061ceae9bdba8a1191899ead70b8fe0eb8cf73e9d29a2ef3a
+  metadata.gz: 958cb7be80dfc2293e00ad6ca4f9021c12e6e817871f7eb0f25c4ead5aa9a10fca8b42db16b45d01e998cd39980545e30087f5a8064ba57c3e3acb07f07c4934
+  data.tar.gz: 7c6e163ad4948037a50c88365407e2b91dd4217d0e3effc9c21d79badbbcd93f823e0695997edf791ade3ffe0e7f67e0b4bf61dde91729a1a62fdc4de42ba7c8

data/3scale_toolbox.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'webmock', '~> 3.4'
   spec.required_ruby_version = '>= 2.5'
 
-  spec.add_dependency '3scale-api', '~> 1.1'
+  spec.add_dependency '3scale-api', '~> 1.4'
   spec.add_dependency 'cri', '~> 2.15'
   spec.add_dependency 'json-schema', '~> 2.8'
   spec.add_dependency 'oas_parser', '~> 0.20'

data/README.md

@@ -13,6 +13,7 @@
    * [Copy service](docs/copy-service.md)
    * [Copy backend](docs/copy-backend.md)
    * [Copy product](docs/copy-product.md)
+   * [Export/Import product](docs/export-import-product.md)
    * [Import from CSV](docs/import-csv.md)
    * [Import from OpenAPI definition](docs/openapi.md)
    * [Export/Import Application Plan](docs/export-import-app-plan.md)
@@ -21,9 +22,10 @@
    * Create, Apply, List, Delete [Method](docs/method.md)
    * Create, Apply, List, Show, Delete [Service](docs/service.md)
    * Create, Apply, List, Delete [ActiveDocs](docs/activedocs.md)
-   * List, Show, Promote, Export [Proxy Configuration](docs/proxy-config.md)
+   * List, Show, Promote, Export, Deploy [Proxy Configuration](docs/proxy-config.md)
    * [Copy Policy Registry](docs/copy-policy-registry.md)
    * Create, Apply, List, Show, Delete, Suspend, Resume [Applications](docs/applications.md)
+   * [Export/Import Product Policy Chain](docs/export-import-policy-chain.md)
    * [Remotes](docs/remotes.md)
 * [Development](#development)
    * [Testing](#testing)
@@ -39,6 +41,7 @@ Supported Ruby interpreters
 
 * MRI 2.5
 * MRI 2.6
+* MRI 2.7
 
 ## Installation
 Install the toolbox:
@@ -120,7 +123,7 @@ rake spec:integration
 
 Integration tests can be run locally or against a real 3scale account.
 When details of the account are set via environment variables,
-integration tests are run agains given account.
+integration tests are run against the given account.
 Otherwise, tests are run locally with mocked 3scale clients.
 
 The easiest way to set everything up is it to have a `.env` file in the root of the project with the following environment variables (set your own values):
@@ -163,7 +166,7 @@ end
 A few things worth highlighting:
 - Your module must include the *ThreeScaleToolbox::Command* module. It allows your command to be added to the toobox command tree.
 - You must implement the `command` module function and return an instance of `Cri::Command` from [cri](https://github.com/ddfreyne/cri)
-- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from remote list and client instantiation is done out of the box.
+- `threescale_client` helper method returns *3scale API* client instance. All the process remote parsing, fetching from the remote list and client instantiation is done out of the box.
 
 Then register the core command in `lib/3scale_toolbox/commands.rb`
 ```

data/lib/3scale_toolbox.rb

@@ -10,15 +10,18 @@ require 'oas_parser'
 require '3scale/api'
 require 'json-schema'
 require 'erb'
+require 'logger'
 
 require '3scale_toolbox/version'
 require '3scale_toolbox/helper'
 require '3scale_toolbox/error'
+require '3scale_toolbox/remote_cache'
 require '3scale_toolbox/proxy_logger'
 require '3scale_toolbox/resource_reader'
 require '3scale_toolbox/configuration'
 require '3scale_toolbox/remotes'
 require '3scale_toolbox/3scale_client_factory'
+require '3scale_toolbox/crds'
 require '3scale_toolbox/entities'
 require '3scale_toolbox/attribute_filters'
 require '3scale_toolbox/base_command'

data/lib/3scale_toolbox/3scale_client_factory.rb

@@ -22,10 +22,9 @@ module ThreeScaleToolbox
         remote = remotes.fetch(remote_str)
       end
 
-      client = remote_client(remote.merge(verify_ssl: verify_ssl))
-      return ProxyLogger.new(client) if verbose
-
-      client
+      client = remote_client(**remote.merge(verify_ssl: verify_ssl))
+      client = ProxyLogger.new(client) if verbose
+      RemoteCache.new(client)
     end
 
     private

data/lib/3scale_toolbox/commands.rb

@@ -15,6 +15,7 @@ require '3scale_toolbox/commands/policy_registry_command'
 require '3scale_toolbox/commands/application_command'
 require '3scale_toolbox/commands/backend_command'
 require '3scale_toolbox/commands/product_command'
+require '3scale_toolbox/commands/policies_command'
 
 module ThreeScaleToolbox
   module Commands
@@ -35,6 +36,7 @@ module ThreeScaleToolbox
       ThreeScaleToolbox::Commands::ApplicationCommand,
       ThreeScaleToolbox::Commands::BackendCommand,
       ThreeScaleToolbox::Commands::ProductCommand,
+      ThreeScaleToolbox::Commands::PoliciesCommand,
     ].freeze
   end
 end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_mapping_rules_task.rb

@@ -7,42 +7,26 @@ module ThreeScaleToolbox
 
           # entrypoint
           def run
-            missing_rules = missing_mapping_rules(source_backend.mapping_rules,
-                                                  target_backend.mapping_rules, metrics_map)
             missing_rules.each do |mapping_rule|
-              mapping_rule.metric_id = metrics_map.fetch(mapping_rule.metric_id)
-              Entities::BackendMappingRule.create(backend: target_backend,
-                                                  attrs: mapping_rule.attrs)
+              mr_attrs = mapping_rule.attrs.merge('metric_id' => metrics_map.fetch(mapping_rule.metric_id))
+              Entities::BackendMappingRule.create(backend: target_backend, attrs: mr_attrs)
             end
-            puts "created #{missing_rules.size} mapping rules"
+            logger.info "created #{missing_rules.size} mapping rules"
+            report['missing_mapping_rules_created'] = missing_rules.size
           end
 
           private
 
           def metrics_map
-            @metrics_map ||= build_metrics_mapping
+            @metrics_map ||= source_backend.metrics_mapping(target_backend)
           end
 
-          def build_metrics_mapping
-            target_mm = target_metrics + target_methods
-            source_mm = source_metrics + source_methods
-            target_mm.map do |target|
-              source = source_mm.find do |m|
-                m.system_name == target.system_name
-              end
-              next if source.nil?
-
-              [source.id, target.id]
-            end.compact.to_h
-          end
-
-          def missing_mapping_rules(source_rules, target_rules, metrics_map)
-            ThreeScaleToolbox::Helper.array_difference(source_rules, target_rules) do |source, target|
-              # map metric_id to the target backend domain
-              source_attrs = source.attrs.merge('metric_id' => metrics_map.fetch(source.metric_id))
-              ThreeScaleToolbox::Helper.compare_hashes(source_attrs,
-                                                       target.attrs,
-                                                       %w[pattern http_method delta metric_id])
+          def missing_rules
+            @missing_rules ||= ThreeScaleToolbox::Helper.array_difference(source_backend.mapping_rules, target_backend.mapping_rules) do |source_rule, target_rule|
+              source_rule.pattern == target_rule.pattern &&
+                source_rule.http_method == target_rule.http_method &&
+                source_rule.delta == target_rule.delta &&
+                metrics_map.fetch(source_rule.metric_id) == target_rule.metric_id
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_methods_task.rb

@@ -8,28 +8,23 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_methods.each(&method(:create_method))
-            puts "created #{missing_methods.size} missing methods"
-            invalidate_target_methods if missing_methods.size.positive?
+            logger.info "created #{missing_methods.size} missing methods"
+            report['missing_methods_created'] = missing_methods.size
           end
 
           private
 
           def create_method(method)
-            # return silently if target metric hits does not exist
-            return if target_hits.nil?
-
-            Entities::BackendMethod.create(backend: target_backend,
-                                           parent_id: target_hits.id,
-                                           attrs:  method.attrs)
+            Entities::BackendMethod.create(backend: target_backend, attrs:  method.attrs)
           rescue ThreeScaleToolbox::ThreeScaleApiError => e
             raise e unless ThreeScaleToolbox::Helper.system_name_already_taken_error?(e.apierrors)
 
-            warn "[WARN] backend method #{method.attrs.fetch('system_name')} not created. " \
+            warn "[WARN] backend method #{method.system_name} not created. " \
               'Backend metric with the same system_name exists.'
           end
 
           def missing_methods
-            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_methods, target_methods) do |source, target|
+            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_backend.methods, target_backend.methods) do |source, target|
               source.system_name == target.system_name
             end
           end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_metrics_task.rb

@@ -8,8 +8,8 @@ module ThreeScaleToolbox
           # entrypoint
           def run
             missing_metrics.each(&method(:create_metric))
-            puts "created #{missing_metrics.size} missing metrics"
-            invalidate_target_metrics if missing_metrics.size.positive?
+            logger.info "created #{missing_metrics.size} missing metrics"
+            report['missing_metrics_created'] = missing_metrics.size
           end
 
           private
@@ -19,8 +19,8 @@ module ThreeScaleToolbox
           end
 
           def missing_metrics
-            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_metrics, target_metrics) do |source, target|
-              source.system_name == target.system_name
+            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_backend.metrics, target_backend.metrics) do |s_m, t_m|
+              s_m.system_name == t_m.system_name
             end
           end
         end

data/lib/3scale_toolbox/commands/backend_command/copy_command/create_or_update_target_backend_task.rb

@@ -23,8 +23,9 @@ module ThreeScaleToolbox
             # assign target backend for other tasks to have it available
             self.target_backend = backend
 
-            puts "source backend ID: #{source_backend.id} system_name: #{source_backend.attrs['system_name']}"
-            puts "target backend ID: #{target_backend.id} system_name: #{target_backend.attrs['system_name']}"
+            logger.info "source backend ID: #{source_backend.id} system_name: #{source_backend.system_name}"
+            logger.info "target backend ID: #{target_backend.id} system_name: #{target_backend.system_name}"
+            report['backend_id'] = target_backend.id
           end
 
           def create_attrs

data/lib/3scale_toolbox/commands/backend_command/copy_command/task.rb

@@ -26,38 +26,6 @@ module ThreeScaleToolbox
             context[:source_backend] ||= find_source_backend
           end
 
-          def source_methods
-            context[:source_methods] ||= source_backend.methods(source_hits)
-          end
-
-          def source_hits
-            context[:source_hits] ||= source_backend.hits
-          end
-
-          def source_metrics
-            context[:source_metrics] ||= source_backend.metrics
-          end
-
-          def target_metrics
-            context[:target_metrics] ||= target_backend.metrics
-          end
-
-          def target_hits
-            context[:target_hits] ||= target_backend.hits
-          end
-
-          def target_methods
-            context[:target_methods] ||= target_backend.methods(target_hits)
-          end
-
-          def invalidate_target_methods
-            context[:target_methods] = nil
-          end
-
-          def invalidate_target_metrics
-            context[:target_metrics] = nil
-          end
-
           def source_remote
             context[:source_remote]
           end
@@ -75,6 +43,16 @@ module ThreeScaleToolbox
             context[:option_target_system_name]
           end
 
+          def logger
+            context[:logger] ||= Logger.new($stdout).tap do |logger|
+              logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+            end
+          end
+
+          def report
+            context[:report] ||= {}
+          end
+
           private
 
           def find_source_backend

data/lib/3scale_toolbox/commands/import_command/issuer_type_transformer.rb

@@ -0,0 +1,16 @@
+module ThreeScaleToolbox
+  module Commands
+    module ImportCommand
+      module OpenAPI
+        # Helper class to validate values for the oidc-issuer-type argument of the import openapi command
+        class IssuerTypeTransformer
+          def call(issuer_type)
+            raise unless %w[rest keycloak].include?(issuer_type)
+
+            issuer_type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command/openapi.rb

@@ -9,6 +9,7 @@ require '3scale_toolbox/commands/import_command/openapi/create_activedocs_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_proxy_step'
 require '3scale_toolbox/commands/import_command/openapi/update_service_oidc_conf_step'
 require '3scale_toolbox/commands/import_command/openapi/update_policies_step'
+require '3scale_toolbox/commands/import_command/issuer_type_transformer'
 
 module ThreeScaleToolbox
   module Commands
@@ -30,6 +31,7 @@ module ThreeScaleToolbox
               flag    nil, 'activedocs-hidden', 'Create ActiveDocs in hidden state'
               flag    nil, 'skip-openapi-validation', 'Skip OpenAPI schema validation'
               flag    nil, 'prefix-matching', 'Use prefix matching instead of strict matching on mapping rules derived from openapi operations'
+              option  nil, 'oidc-issuer-type', 'OIDC Issuer Type (rest, keycloak)', argument: :required, transform: IssuerTypeTransformer.new            
               option  nil, 'oidc-issuer-endpoint', 'OIDC Issuer Endpoint', argument: :required
               option  nil, 'default-credentials-userkey', 'Default credentials policy userkey', argument: :required
               option  nil, 'override-private-basepath', 'Override the basepath for the private URLs', argument: :required
@@ -77,6 +79,7 @@ module ThreeScaleToolbox
               threescale_client: threescale_client(fetch_required_option(:destination)),
               target_system_name: options[:target_system_name],
               activedocs_published: !options[:'activedocs-hidden'],
+              oidc_issuer_type: options[:'oidc-issuer-type'],
               oidc_issuer_endpoint: options[:'oidc-issuer-endpoint'],
               default_credentials_userkey: options[:'default-credentials-userkey'],
               skip_openapi_validation: options[:'skip-openapi-validation'],

data/lib/3scale_toolbox/commands/import_command/openapi/create_mapping_rule_step.rb

@@ -7,7 +7,8 @@ module ThreeScaleToolbox
 
           def call
             operations.each do |op|
-              service.create_mapping_rule(op.mapping_rule)
+              Entities::MappingRule.create(service: service,
+                                           attrs: op.mapping_rule)
               puts "Created #{op.http_method} #{op.pattern} endpoint"
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_method_step.rb

@@ -7,18 +7,13 @@ module ThreeScaleToolbox
 
           def call
             missing_operations.each do |op|
-              method = Entities::Method.create(service: service, parent_id: hits_metric_id,
-                                               attrs: op.method)
+              method = Entities::Method.create(service: service, attrs: op.method)
               op.set(:metric_id, method.id)
             end
 
             existing_operations.each do |op|
               method_attrs = methods_index.fetch(op.method['system_name'])
-              method = Entities::Method.new(
-                id: method_attrs.fetch('id'),
-                parent_id: hits_metric_id,
-                service: service
-              )
+              method = Entities::Method.new(id: method_attrs.fetch('id'), service: service)
               method.update(op.method)
               op.set(:metric_id, method.id)
             end
@@ -26,12 +21,8 @@ module ThreeScaleToolbox
 
           private
 
-          def hits_metric_id
-            @hits_metric_id ||= service.hits['id']
-          end
-
           def methods_index
-            @methods_index ||= service.methods(hits_metric_id).each_with_object({}) do |method, acc|
+            @methods_index ||= service.methods.each_with_object({}) do |method, acc|
               acc[method['system_name']] = method
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi/step.rb

@@ -57,6 +57,10 @@ module ThreeScaleToolbox
             context[:api_spec_resource]
           end
 
+          def oidc_issuer_type
+            context[:oidc_issuer_type]
+          end
+
           def oidc_issuer_endpoint
             context[:oidc_issuer_endpoint]
           end

data/lib/3scale_toolbox/commands/import_command/openapi/update_service_proxy_step.rb

@@ -52,6 +52,7 @@ module ThreeScaleToolbox
             case (type = api_spec.security[:type])
             when 'oauth2'
               settings[:credentials_location] = 'headers'
+              settings[:oidc_issuer_type] = oidc_issuer_type unless oidc_issuer_type.nil?
               settings[:oidc_issuer_endpoint] = oidc_issuer_endpoint unless oidc_issuer_endpoint.nil?
             when 'apiKey'
               settings[:credentials_location] = credentials_location

data/lib/3scale_toolbox/commands/methods_command/apply_command.rb

@@ -47,11 +47,9 @@ module ThreeScaleToolbox
           def run
             validate_option_params
             hits = service.hits
-            method = Entities::Method.find(service: service, parent_id: hits.fetch('id'),
-                                           ref: method_ref)
+            method = Entities::Method.find(service: service, ref: method_ref)
             if method.nil?
-              method = Entities::Method.create(service: service, parent_id: hits.fetch('id'),
-                                               attrs: create_method_attrs)
+              method = Entities::Method.create(service: service, attrs: create_method_attrs)
             else
               method.update(method_attrs) unless method_attrs.empty?
             end

data/lib/3scale_toolbox/commands/methods_command/create_command.rb

@@ -40,10 +40,8 @@ module ThreeScaleToolbox
           end
 
           def run
-            hits = service.hits
             method = ThreeScaleToolbox::Entities::Method.create(
               service: service,
-              parent_id: hits.fetch('id'),
               attrs: method_attrs
             )
             method.disable if option_disabled

data/lib/3scale_toolbox/commands/methods_command/delete_command.rb

@@ -44,7 +44,7 @@ module ThreeScaleToolbox
 
           def find_method
             hits = service.hits
-            Entities::Method.find(service: service, parent_id: hits.fetch('id'), ref: method_ref).tap do |p|
+            Entities::Method.find(service: service, ref: method_ref).tap do |p|
               raise ThreeScaleToolbox::Error, "Method #{method_ref} does not exist" if p.nil?
             end
           end