3scale_toolbox 0.12.4 → 0.17.0

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_methods_task.rb

@@ -0,0 +1,40 @@
+module ThreeScaleToolbox
+  module Commands
+    module BackendCommand
+      module CopyCommand
+        class CopyMethodsTask
+          include Task
+
+          # entrypoint
+          def run
+            missing_methods.each(&method(:create_method))
+            puts "created #{missing_methods.size} missing methods"
+            invalidate_target_methods if missing_methods.size.positive?
+          end
+
+          private
+
+          def create_method(method)
+            # return silently if target metric hits does not exist
+            return if target_hits.nil?
+
+            Entities::BackendMethod.create(backend: target_backend,
+                                           parent_id: target_hits.id,
+                                           attrs:  method.attrs)
+          rescue ThreeScaleToolbox::ThreeScaleApiError => e
+            raise e unless ThreeScaleToolbox::Helper.system_name_already_taken_error?(e.apierrors)
+
+            warn "[WARN] backend method #{method.attrs.fetch('system_name')} not created. " \
+              'Backend metric with the same system_name exists.'
+          end
+
+          def missing_methods
+            @missing_methods ||= ThreeScaleToolbox::Helper.array_difference(source_methods, target_methods) do |source, target|
+              source.system_name == target.system_name
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/backend_command/copy_command/copy_metrics_task.rb

@@ -0,0 +1,30 @@
+module ThreeScaleToolbox
+  module Commands
+    module BackendCommand
+      module CopyCommand
+        class CopyMetricsTask
+          include Task
+
+          # entrypoint
+          def run
+            missing_metrics.each(&method(:create_metric))
+            puts "created #{missing_metrics.size} missing metrics"
+            invalidate_target_metrics if missing_metrics.size.positive?
+          end
+
+          private
+
+          def create_metric(metric)
+            Entities::BackendMetric.create(backend: target_backend, attrs: metric.attrs)
+          end
+
+          def missing_metrics
+            @missing_metrics ||= ThreeScaleToolbox::Helper.array_difference(source_metrics, target_metrics) do |source, target|
+              source.system_name == target.system_name
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/backend_command/copy_command/create_or_update_target_backend_task.rb

@@ -0,0 +1,45 @@
+module ThreeScaleToolbox
+  module Commands
+    module BackendCommand
+      module CopyCommand
+        class CreateOrUpdateTargetBackendTask
+          include Task
+
+          # entrypoint
+          def run
+            backend = Entities::Backend.find(remote: target_remote, ref: target_backend_ref)
+
+            if backend.nil?
+              backend = Entities::Backend.create(remote: target_remote,
+                                                 attrs: create_attrs)
+            elsif backend == source_backend
+              message = 'source and destination backends are the same: ' \
+                "ID: #{source_backend.id} system_name: #{source_backend.attrs['system_name']}"
+              warn "\e[1m\e[31mWarning: #{message}\e[0m"
+            else
+              backend.update update_attrs
+            end
+
+            # assign target backend for other tasks to have it available
+            self.target_backend = backend
+
+            puts "source backend ID: #{source_backend.id} system_name: #{source_backend.attrs['system_name']}"
+            puts "target backend ID: #{target_backend.id} system_name: #{target_backend.attrs['system_name']}"
+          end
+
+          def create_attrs
+            source_backend.attrs.merge('system_name' => target_backend_ref)
+          end
+
+          def update_attrs
+            source_backend.attrs.merge('system_name' => target_backend_ref)
+          end
+
+          def target_backend_ref
+            option_target_system_name || source_backend.attrs.fetch('system_name')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/backend_command/copy_command/task.rb

@@ -0,0 +1,89 @@
+module ThreeScaleToolbox
+  module Commands
+    module BackendCommand
+      module CopyCommand
+        module Task
+          attr_reader :context
+
+          def initialize(context)
+            @context = context
+          end
+
+          def call
+            run
+          end
+
+          def target_backend=(target)
+            context[:target_backend] = target
+          end
+
+          def target_backend
+            context[:target_backend] ||= raise ThreeScaleToolbox::Error, 'Unexpected error. ' \
+              'Target backend should have been created or updated'
+          end
+
+          def source_backend
+            context[:source_backend] ||= find_source_backend
+          end
+
+          def source_methods
+            context[:source_methods] ||= source_backend.methods(source_hits)
+          end
+
+          def source_hits
+            context[:source_hits] ||= source_backend.hits
+          end
+
+          def source_metrics
+            context[:source_metrics] ||= source_backend.metrics
+          end
+
+          def target_metrics
+            context[:target_metrics] ||= target_backend.metrics
+          end
+
+          def target_hits
+            context[:target_hits] ||= target_backend.hits
+          end
+
+          def target_methods
+            context[:target_methods] ||= target_backend.methods(target_hits)
+          end
+
+          def invalidate_target_methods
+            context[:target_methods] = nil
+          end
+
+          def invalidate_target_metrics
+            context[:target_metrics] = nil
+          end
+
+          def source_remote
+            context[:source_remote]
+          end
+
+          def target_remote
+            context[:target_remote]
+          end
+
+          def source_backend_ref
+            context[:source_backend_ref] ||= raise ThreeScaleToolbox::Error, 'Unexpected error. ' \
+              'source_backend_ref not found'
+          end
+
+          def option_target_system_name
+            context[:option_target_system_name]
+          end
+
+          private
+
+          def find_source_backend
+            Entities::Backend.find(remote: source_remote, ref: source_backend_ref).tap do |backend|
+              raise ThreeScaleToolbox::Error, "Backend #{source_backend_ref} does not exist" if backend.nil?
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/copy_command.rb

@@ -1,4 +1,4 @@
-require '3scale_toolbox/commands/copy_command/copy_service'
+require '3scale_toolbox/commands/copy_command/service_command'
 
 module ThreeScaleToolbox
   module Commands
@@ -16,7 +16,7 @@ module ThreeScaleToolbox
           end
         end
       end
-      add_subcommand(CopyServiceSubcommand)
+      add_subcommand(ServiceSubcommand)
     end
   end
 end

data/lib/3scale_toolbox/commands/copy_command/service_command.rb

@@ -0,0 +1,40 @@
+module ThreeScaleToolbox
+  module Commands
+    module CopyCommand
+      class ServiceSubcommand < Cri::CommandRunner
+        include ThreeScaleToolbox::Command
+
+        def self.command
+          Cri::Command.define do
+            name        'service'
+            usage       'service [opts] -s <src> -d <dst> <source-service>'
+            summary     'copy service'
+            description <<-HEREDOC
+            This command makes a copy of the referenced service.
+            Target service will be searched by source service system name. System name can be overriden with `--target_system_name` option.
+            If a service with the selected `system_name` is not found, it will be created.
+            \n Components of the service being copied:
+            \nservice settings
+            \nproxy settings
+            \npricing rules
+            \nactivedocs
+            \nmetrics
+            \nmethods
+            \napplication plans
+            \nmapping rules
+            HEREDOC
+
+            option  :s, :source, '3scale source instance. Url or remote name', argument: :required
+            option  :d, :destination, '3scale target instance. Url or remote name', argument: :required
+            option  :t, 'target_system_name', 'Target system name. Default to source system name', argument: :required
+            flag    :f, :force, 'Overwrites the mapping rules by deleting all rules from target service first'
+            flag    :r, 'rules-only', 'Only mapping rules are copied'
+            param   :source_service
+
+            runner Commands::ServiceCommand::CopySubcommand
+          end
+        end
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/import_command/openapi.rb

@@ -2,7 +2,6 @@ require '3scale_toolbox/commands/import_command/openapi/method'
 require '3scale_toolbox/commands/import_command/openapi/mapping_rule'
 require '3scale_toolbox/commands/import_command/openapi/operation'
 require '3scale_toolbox/commands/import_command/openapi/step'
-require '3scale_toolbox/commands/import_command/openapi/threescale_api_spec'
 require '3scale_toolbox/commands/import_command/openapi/create_method_step'
 require '3scale_toolbox/commands/import_command/openapi/create_mapping_rule_step'
 require '3scale_toolbox/commands/import_command/openapi/create_service_step'
@@ -30,6 +29,7 @@ module ThreeScaleToolbox
               option  :t, 'target_system_name', 'Target system name', argument: :required
               flag    nil, 'activedocs-hidden', 'Create ActiveDocs in hidden state'
               flag    nil, 'skip-openapi-validation', 'Skip OpenAPI schema validation'
+              flag    nil, 'prefix-matching', 'Use prefix matching instead of strict matching on mapping rules derived from openapi operations'
               option  nil, 'oidc-issuer-endpoint', 'OIDC Issuer Endpoint', argument: :required
               option  nil, 'default-credentials-userkey', 'Default credentials policy userkey', argument: :required
               option  nil, 'override-private-basepath', 'Override the basepath for the private URLs', argument: :required
@@ -37,6 +37,8 @@ module ThreeScaleToolbox
               option  nil, 'staging-public-base-url', 'Custom public staging URL', argument: :required
               option  nil, 'production-public-base-url', 'Custom public production URL', argument: :required
               option  nil, 'override-private-base-url', 'Custom private base URL', argument: :required
+              option nil, 'backend-api-secret-token', 'Custom secret token sent by the API gateway to the backend API',argument: :required
+              option nil, 'backend-api-host-header', 'Custom host header sent by the API gateway to the backend API', argument: :required
               param   :openapi_resource
 
               runner OpenAPISubcommand
@@ -49,7 +51,7 @@ module ThreeScaleToolbox
             # other tasks might read proxy settings (CreateActiveDocsStep does)
             tasks << UpdateServiceProxyStep.new(context)
             tasks << CreateMethodsStep.new(context)
-            tasks << Tasks::DestroyMappingRulesTask.new(context)
+            tasks << ThreeScaleToolbox::Commands::ServiceCommand::CopyCommand::DestroyMappingRulesTask.new(context)
             tasks << CreateMappingRulesStep.new(context)
             tasks << CreateActiveDocsStep.new(context)
             tasks << UpdateServiceOidcConfStep.new(context)
@@ -59,7 +61,7 @@ module ThreeScaleToolbox
             tasks.each(&:call)
 
             # This should be the last step
-            Tasks::BumpProxyVersionTask.new(service: context[:target]).call
+            ThreeScaleToolbox::Commands::ServiceCommand::CopyCommand::BumpProxyVersionTask.new(service: context[:target]).call
           end
 
           private
@@ -69,10 +71,9 @@ module ThreeScaleToolbox
           end
 
           def create_context
-            openapi_resource = load_resource(arguments[:openapi_resource])
             {
               api_spec_resource: openapi_resource,
-              api_spec: ThreeScaleApiSpec.new(load_openapi(openapi_resource), options[:'override-public-basepath']),
+              api_spec: openapi_parser,
               threescale_client: threescale_client(fetch_required_option(:destination)),
               target_system_name: options[:target_system_name],
               activedocs_published: !options[:'activedocs-hidden'],
@@ -80,14 +81,35 @@ module ThreeScaleToolbox
               default_credentials_userkey: options[:'default-credentials-userkey'],
               skip_openapi_validation: options[:'skip-openapi-validation'],
               override_private_basepath: options[:'override-private-basepath'],
+              override_public_basepath: options[:'override-public-basepath'],
               production_public_base_url: options[:'production-public-base-url'],
               staging_public_base_url: options[:'staging-public-base-url'],
               override_private_base_url: options[:'override-private-base-url'],
+              backend_api_secret_token: options[:'backend-api-secret-token'],
+              backend_api_host_header: options[:'backend-api-host-header'],
+              prefix_matching: options[:'prefix-matching'],
+              delete_mapping_rules: true,
             }
           end
 
-          def load_openapi(openapi_resource)
-            Swagger.build(openapi_resource, validate: !options[:'skip-openapi-validation'])
+          def openapi_resource
+            @openapi_resource ||= load_resource(openapi_path)
+          end
+
+          def openapi_path
+            arguments[:openapi_resource]
+          end
+
+          def validate
+            !options[:'skip-openapi-validation']
+          end
+
+          def openapi_parser
+            if openapi_resource.key?('openapi')
+              ThreeScaleToolbox::OpenAPI::OAS3.build(openapi_path, openapi_resource, validate: validate)
+            else
+              ThreeScaleToolbox::OpenAPI::Swagger.build(openapi_resource, validate: validate)
+            end
           rescue JSON::Schema::ValidationError => e
             raise ThreeScaleToolbox::Error, "OpenAPI schema validation failed: #{e.message}"
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_activedocs_step.rb

@@ -53,26 +53,13 @@ module ThreeScaleToolbox
             # Other processing steps can work with original openapi spec
             Helper.hash_deep_dup(resource).tap do |activedocs|
               # public production base URL
-              URI(service.proxy.fetch('endpoint')).tap do |uri|
-                activedocs['host'] = "#{uri.host}:#{uri.port}"
-                activedocs['schemes'] = [uri.scheme]
-              end
-
               # the basePath field is updated to a new value only when overriden by optional param
-              activedocs['basePath'] = api_spec.public_base_path
-
+              api_spec.set_server_url(activedocs,
+                                      URI.join(service.proxy.fetch('endpoint'), public_base_path))
               # security definitions
               # just valid for oauth2 when oidc_issuer_endpoint is supplied
-              if !security.nil? && security.type == 'oauth2' && !oidc_issuer_endpoint.nil?
-                # authorizationURL
-                if %w[implicit accessCode].include?(security.flow)
-                  activedocs['securityDefinitions'][security.id]['authorizationUrl'] = authorization_url
-                end
-
-                # tokenUrl
-                if %w[password application accessCode].include?(security.flow)
-                  activedocs['securityDefinitions'][security.id]['tokenUrl'] = token_url
-                end
+              if !api_spec.security.nil? && api_spec.security[:type] == 'oauth2' && !oidc_issuer_endpoint.nil?
+                api_spec.set_oauth2_urls(activedocs, api_spec.security[:id], authorization_url, token_url)
               end
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_service_step.rb

@@ -33,7 +33,7 @@ module ThreeScaleToolbox
             default_service_settings.tap do |svc|
               svc['name'] = service_name
               svc['description'] = service_description
-              svc['backend_version'] = backend_version
+              svc['backend_version'] = api_spec.service_backend_version
               svc['system_name'] = service_system_name
               svc['deployment_option'] = 'self_managed' if !production_public_base_url.nil? || !staging_public_base_url.nil?
             end
@@ -50,10 +50,6 @@ module ThreeScaleToolbox
           def service_description
             api_spec.description
           end
-
-          def backend_version
-            api_spec.backend_version
-          end
         end
       end
     end

data/lib/3scale_toolbox/commands/import_command/openapi/mapping_rule.rb

@@ -17,8 +17,9 @@ module ThreeScaleToolbox
           end
 
           def pattern
-            # apply strict matching
-            "#{raw_pattern}$"
+            res = "#{raw_pattern}"
+            res = "#{res}$" if !operation[:prefix_matching] # apply strict matching
+            res
           end
 
           def raw_pattern

data/lib/3scale_toolbox/commands/import_command/openapi/step.rb

@@ -31,7 +31,21 @@ module ThreeScaleToolbox
             # api_spec.operations are readonly
             # store operations in context
             # each operation can be extended with extra information to be used later
-            context[:operations] ||= api_spec.operations
+            context[:operations] ||= build_3scale_operations
+          end
+
+          def build_3scale_operations
+            api_spec.operations.map do |op|
+              Operation.new(
+                base_path: base_path,
+                public_base_path: public_base_path,
+                path: op[:path],
+                verb: op[:verb],
+                operationId: op[:operation_id],
+                description: op[:description],
+                prefix_matching: prefix_matching,
+              )
+            end
           end
 
           def target_system_name
@@ -43,10 +57,6 @@ module ThreeScaleToolbox
             context[:api_spec_resource]
           end
 
-          def security
-            api_spec.security
-          end
-
           def oidc_issuer_endpoint
             context[:oidc_issuer_endpoint]
           end
@@ -59,6 +69,10 @@ module ThreeScaleToolbox
             context[:override_private_basepath]
           end
 
+          def override_public_basepath
+            context[:override_public_basepath]
+          end
+
           def production_public_base_url
             context[:production_public_base_url]
           end
@@ -70,6 +84,30 @@ module ThreeScaleToolbox
           def override_private_base_url
             context[:override_private_base_url]
           end
+
+          def backend_api_secret_token
+            context[:backend_api_secret_token]
+          end
+
+          def backend_api_host_header
+            context[:backend_api_host_header]
+          end
+
+          def prefix_matching
+            context[:prefix_matching]
+          end
+
+          def base_path
+            api_spec.base_path || '/'
+          end
+
+          def public_base_path
+            override_public_basepath || base_path
+          end
+
+          def private_base_path
+            override_private_basepath || base_path
+          end
         end
       end
     end