3scale_toolbox 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f7c696659a170ef2b061871c02cae7bf03fee297f0a9820e9ad8edd13d273e6
-  data.tar.gz: 8452cce3946098d1732184b7e371c1bf80cfc16071608eba39250ac318f40b0d
+  metadata.gz: bf0d2a187e793d654f6e6cef6990cfabaada5072f3a72b9820b1fed1ac13ddfb
+  data.tar.gz: c1b56e8e76059e0143444ff9ef12da633b97ef74b0fccb6875480be8b22cf9e6
 SHA512:
-  metadata.gz: 76a2ed5ef4e2e987a8db94d586fffe2d75e6c84408894a74d75a0d7050739ca70339f4203b2cd883f5670bfda87d4bfd7a8bfa5a7bbbc6eb024de565be5a3333
-  data.tar.gz: 38fba12044b61af86f71765c4e7f58ae1c0a65c73aaa491fe7c696f6ca0d58b84cb14a5651345d8cf9e7978cb7ae2e5443e21404bb04539fa93b315daea4a363
+  metadata.gz: f2bf8c3a1f5860a3c6d24fa5fbc976b9b3361ea8ef6a57bbc0707d5f3900a31d8022e9596080bf8129d47116d13d3ab4d2ae7bfc41e246d179adf0f757c7c598
+  data.tar.gz: 3a206838d82a662f56276e7d98291cf74f5bc69eb9f39de239bb3d15f457e62b81e336ee1d0438a508fdecc46f692b49610627ef6413d05788b4c9f1a8c3f2e5

data/3scale_toolbox.gemspec

@@ -34,9 +34,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 3.8'
   spec.add_development_dependency 'webmock', '~> 3.4'
-  spec.required_ruby_version = '>= 2.4'
+  spec.required_ruby_version = '>= 2.5'
 
   spec.add_dependency '3scale-api', '~> 0.6.0'
   spec.add_dependency 'cri', '~> 2.15'
   spec.add_dependency 'json-schema', '~> 2.8'
+  spec.add_dependency 'oas_parser', '~> 0.20'
 end

data/README.md

@@ -1,5 +1,5 @@
 # 3scale toolbox
-[![Build Status](https://travis-ci.org/3scale/3scale_toolbox.svg?branch=master)](https://travis-ci.org/3scale/3scale_toolbox)
+[![CircleCI](https://circleci.com/gh/3scale/3scale_toolbox.svg?style=svg)](https://circleci.com/gh/3scale/3scale_toolbox)
 
 This software is licensed under the [Apache 2.0 license](https://www.apache.org/licenses/LICENSE-2.0).
 
@@ -9,6 +9,7 @@ See the LICENSE and NOTICE files that should have been provided along with this
 3scale toolbox is a set of tools to help you manage your 3scale product. Using the [3scale API Ruby Client](https://github.com/3scale/3scale-api-ruby).
 
 ## Table of contents
+* [Requirements](#requirements)
 * [Installation](#installation)
 * [Usage](#usage)
    * [Copy a service](docs/copy-service.md)
@@ -32,10 +33,27 @@ See the LICENSE and NOTICE files that should have been provided along with this
 * [Troubleshooting](#troubleshooting)
 * [Contributing](#contributing)
 
+## Requirements
+Supported Ruby interpreters
+
+* MRI 2.5
+* MRI 2.6
+
 ## Installation
 Install the toolbox:
 
-    $ gem install 3scale_toolbox
+```
+$ gem install 3scale_toolbox
+```
+
+The [3scale toolbox packaging repo](https://github.com/3scale/3scale_toolbox_packaging)
+provides packages and installation/deployment steps for the following platforms:
+* CentOS/Fedora
+* Ubuntu/Debian
+* Mac OS X
+* Windows
+* Docker
+* Kubernetes / Openshift
 
 ## Usage
 
@@ -178,7 +196,7 @@ $ 3scale service_list my-3scale-instance
 
 It is a requirement that we include a file describing all the licenses used in the product, so that users can examine it.
 
-Run `rake license_finder:check` to check licenses when dependencies change. 
+Run `rake license_finder:check` to check licenses when dependencies change.
 
 Run `rake license_finder:report > licenses.xml` to update licenses file.
 

data/lib/3scale_toolbox.rb

@@ -6,20 +6,24 @@ require 'time'
 require 'csv'
 require 'net/http'
 require 'pathname'
+require 'oas_parser'
 require '3scale/api'
+require 'json-schema'
+require 'erb'
 
 require '3scale_toolbox/version'
 require '3scale_toolbox/helper'
 require '3scale_toolbox/error'
 require '3scale_toolbox/proxy_logger'
 require '3scale_toolbox/resource_reader'
-require '3scale_toolbox/swagger'
 require '3scale_toolbox/configuration'
 require '3scale_toolbox/remotes'
 require '3scale_toolbox/3scale_client_factory'
 require '3scale_toolbox/entities'
 require '3scale_toolbox/tasks'
+require '3scale_toolbox/attribute_filters'
 require '3scale_toolbox/base_command'
+require '3scale_toolbox/openapi'
 require '3scale_toolbox/commands'
 require '3scale_toolbox/cli'
 

data/lib/3scale_toolbox/attribute_filters.rb

@@ -0,0 +1,2 @@
+require '3scale_toolbox/attribute_filters/attribute_filter'
+require '3scale_toolbox/attribute_filters/service_id_from_ref_filter'

data/lib/3scale_toolbox/attribute_filters/attribute_filter.rb

@@ -0,0 +1,9 @@
+module ThreeScaleToolbox
+  module AttributeFilters
+    module AttributeFilter
+      def filter(enumerable)
+        raise "Not implemented"
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/attribute_filters/service_id_from_ref_filter.rb

@@ -0,0 +1,30 @@
+module ThreeScaleToolbox
+  module AttributeFilters
+    class ServiceIDFilterFromServiceRef
+      include AttributeFilter
+
+      attr_reader :remote, :service_ref, :service_id_key
+
+      def initialize(remote, service_ref, service_id_key)
+        @remote = remote
+        @service_ref = service_ref
+        @service_id_key = service_id_key
+      end
+
+      def filter(enumerable)
+        svc_id = find_service
+        enumerable.select { |e| e.key?(service_id_key) && e[service_id_key].to_s == svc_id.to_s }
+      end
+
+      private
+
+      def find_service
+        svc_id = -1
+        Entities::Service.find(remote: remote, ref: service_ref).tap do |svc|
+          svc_id = svc.id if !svc.nil?
+        end
+        svc_id
+      end
+    end
+  end
+end

data/lib/3scale_toolbox/commands/activedocs_command/apply_command.rb

@@ -20,7 +20,7 @@ module ThreeScaleToolbox
               option :i, :'service-id', "Specify the Service ID associated to the ActiveDocs", argument: :required
               option :p, :'publish', "Specify it to publish the ActiveDocs on the Developer Portal. Otherwise it will be hidden", argument: :forbidden
               option nil, :'hide', "Specify it to hide the ActiveDocs on the Developer Portal", argument: :forbidden
-              option nil, :'skip-swagger-validations', "Specify it to skip validation of the Swagger specification", argument: :forbidden
+              option nil, :'skip-swagger-validations', "Skip validation of the Swagger specification. true or false", argument: :required, transform: ThreeScaleToolbox::Helper::BooleanTransformer.new
               option :d, :'description', "Specify the description of the ActiveDocs", argument: :required
               option :s, :'name', "Specify the name of the ActiveDocs", argument: :required
               option nil, :'openapi-spec', "Specify the swagger spec. Can be a file, an URL or '-' to read from stdin. This option is mandatory when applying the ActiveDoc for the first time", argument: :required

data/lib/3scale_toolbox/commands/activedocs_command/list_command.rb

@@ -14,6 +14,7 @@ module ThreeScaleToolbox
               runner ListSubcommand
 
               param :remote
+              option :s, :'service-ref', "Filter the ActiveDocs by Service reference", argument: :required
             end
           end
 
@@ -42,8 +43,24 @@ module ThreeScaleToolbox
             puts fields_to_show.map { |e| e.upcase }.join("\t")
           end
 
+          def service_ref_filter
+            options[:'service-ref']
+          end
+
+          def filters
+            res = []
+            if !service_ref_filter.nil?
+              res << AttributeFilters::ServiceIDFilterFromServiceRef.new(remote, service_ref_filter, "service_id")
+            end
+            res
+          end
+
+          def filtered_activedocs(activedocs)
+            filters.reduce(activedocs) { |current_list, filter| filter.filter(current_list) }
+          end
+
           def print_results(activedocs, fields_to_show)
-            activedocs.each do |activedoc|
+            filtered_activedocs(activedocs).each do |activedoc|
               puts fields_to_show.map { |field| activedoc.fetch(field, '(empty)') }.join("\t")
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi.rb

@@ -2,7 +2,6 @@ require '3scale_toolbox/commands/import_command/openapi/method'
 require '3scale_toolbox/commands/import_command/openapi/mapping_rule'
 require '3scale_toolbox/commands/import_command/openapi/operation'
 require '3scale_toolbox/commands/import_command/openapi/step'
-require '3scale_toolbox/commands/import_command/openapi/threescale_api_spec'
 require '3scale_toolbox/commands/import_command/openapi/create_method_step'
 require '3scale_toolbox/commands/import_command/openapi/create_mapping_rule_step'
 require '3scale_toolbox/commands/import_command/openapi/create_service_step'
@@ -30,6 +29,7 @@ module ThreeScaleToolbox
               option  :t, 'target_system_name', 'Target system name', argument: :required
               flag    nil, 'activedocs-hidden', 'Create ActiveDocs in hidden state'
               flag    nil, 'skip-openapi-validation', 'Skip OpenAPI schema validation'
+              flag    nil, 'prefix-matching', 'Use prefix matching instead of strict matching on mapping rules derived from openapi operations'
               option  nil, 'oidc-issuer-endpoint', 'OIDC Issuer Endpoint', argument: :required
               option  nil, 'default-credentials-userkey', 'Default credentials policy userkey', argument: :required
               option  nil, 'override-private-basepath', 'Override the basepath for the private URLs', argument: :required
@@ -37,6 +37,8 @@ module ThreeScaleToolbox
               option  nil, 'staging-public-base-url', 'Custom public staging URL', argument: :required
               option  nil, 'production-public-base-url', 'Custom public production URL', argument: :required
               option  nil, 'override-private-base-url', 'Custom private base URL', argument: :required
+              option nil, 'backend-api-secret-token', 'Custom secret token sent by the API gateway to the backend API',argument: :required
+              option nil, 'backend-api-host-header', 'Custom host header sent by the API gateway to the backend API', argument: :required
               param   :openapi_resource
 
               runner OpenAPISubcommand
@@ -69,10 +71,9 @@ module ThreeScaleToolbox
           end
 
           def create_context
-            openapi_resource = load_resource(arguments[:openapi_resource])
             {
               api_spec_resource: openapi_resource,
-              api_spec: ThreeScaleApiSpec.new(load_openapi(openapi_resource), options[:'override-public-basepath']),
+              api_spec: openapi_parser,
               threescale_client: threescale_client(fetch_required_option(:destination)),
               target_system_name: options[:target_system_name],
               activedocs_published: !options[:'activedocs-hidden'],
@@ -80,14 +81,34 @@ module ThreeScaleToolbox
               default_credentials_userkey: options[:'default-credentials-userkey'],
               skip_openapi_validation: options[:'skip-openapi-validation'],
               override_private_basepath: options[:'override-private-basepath'],
+              override_public_basepath: options[:'override-public-basepath'],
               production_public_base_url: options[:'production-public-base-url'],
               staging_public_base_url: options[:'staging-public-base-url'],
               override_private_base_url: options[:'override-private-base-url'],
+              backend_api_secret_token: options[:'backend-api-secret-token'],
+              backend_api_host_header: options[:'backend-api-host-header'],
+              prefix_matching: options[:'prefix-matching'],
             }
           end
 
-          def load_openapi(openapi_resource)
-            Swagger.build(openapi_resource, validate: !options[:'skip-openapi-validation'])
+          def openapi_resource
+            @openapi_resource ||= load_resource(openapi_path)
+          end
+
+          def openapi_path
+            arguments[:openapi_resource]
+          end
+
+          def validate
+            !options[:'skip-openapi-validation']
+          end
+
+          def openapi_parser
+            if openapi_resource.key?('openapi')
+              ThreeScaleToolbox::OpenAPI::OAS3.build(openapi_path, openapi_resource, validate: validate)
+            else
+              ThreeScaleToolbox::OpenAPI::Swagger.build(openapi_resource, validate: validate)
+            end
           rescue JSON::Schema::ValidationError => e
             raise ThreeScaleToolbox::Error, "OpenAPI schema validation failed: #{e.message}"
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_activedocs_step.rb

@@ -53,26 +53,13 @@ module ThreeScaleToolbox
             # Other processing steps can work with original openapi spec
             Helper.hash_deep_dup(resource).tap do |activedocs|
               # public production base URL
-              URI(service.proxy.fetch('endpoint')).tap do |uri|
-                activedocs['host'] = "#{uri.host}:#{uri.port}"
-                activedocs['schemes'] = [uri.scheme]
-              end
-
               # the basePath field is updated to a new value only when overriden by optional param
-              activedocs['basePath'] = api_spec.public_base_path
-
+              api_spec.set_server_url(activedocs,
+                                      URI.join(service.proxy.fetch('endpoint'), public_base_path))
               # security definitions
               # just valid for oauth2 when oidc_issuer_endpoint is supplied
-              if !security.nil? && security.type == 'oauth2' && !oidc_issuer_endpoint.nil?
-                # authorizationURL
-                if %w[implicit accessCode].include?(security.flow)
-                  activedocs['securityDefinitions'][security.id]['authorizationUrl'] = authorization_url
-                end
-
-                # tokenUrl
-                if %w[password application accessCode].include?(security.flow)
-                  activedocs['securityDefinitions'][security.id]['tokenUrl'] = token_url
-                end
+              if !api_spec.security.nil? && api_spec.security[:type] == 'oauth2' && !oidc_issuer_endpoint.nil?
+                api_spec.set_oauth2_urls(activedocs, api_spec.security[:id], authorization_url, token_url)
               end
             end
           end

data/lib/3scale_toolbox/commands/import_command/openapi/create_service_step.rb

@@ -33,7 +33,7 @@ module ThreeScaleToolbox
             default_service_settings.tap do |svc|
               svc['name'] = service_name
               svc['description'] = service_description
-              svc['backend_version'] = backend_version
+              svc['backend_version'] = api_spec.service_backend_version
               svc['system_name'] = service_system_name
               svc['deployment_option'] = 'self_managed' if !production_public_base_url.nil? || !staging_public_base_url.nil?
             end
@@ -50,10 +50,6 @@ module ThreeScaleToolbox
           def service_description
             api_spec.description
           end
-
-          def backend_version
-            api_spec.backend_version
-          end
         end
       end
     end

data/lib/3scale_toolbox/commands/import_command/openapi/mapping_rule.rb

@@ -17,8 +17,9 @@ module ThreeScaleToolbox
           end
 
           def pattern
-            # apply strict matching
-            "#{raw_pattern}$"
+            res = "#{raw_pattern}"
+            res = "#{res}$" if !operation[:prefix_matching] # apply strict matching
+            res
           end
 
           def raw_pattern

data/lib/3scale_toolbox/commands/import_command/openapi/step.rb

@@ -31,7 +31,21 @@ module ThreeScaleToolbox
             # api_spec.operations are readonly
             # store operations in context
             # each operation can be extended with extra information to be used later
-            context[:operations] ||= api_spec.operations
+            context[:operations] ||= build_3scale_operations
+          end
+
+          def build_3scale_operations
+            api_spec.operations.map do |op|
+              Operation.new(
+                base_path: base_path,
+                public_base_path: public_base_path,
+                path: op[:path],
+                verb: op[:verb],
+                operationId: op[:operation_id],
+                description: op[:description],
+                prefix_matching: prefix_matching,
+              )
+            end
           end
 
           def target_system_name
@@ -43,10 +57,6 @@ module ThreeScaleToolbox
             context[:api_spec_resource]
           end
 
-          def security
-            api_spec.security
-          end
-
           def oidc_issuer_endpoint
             context[:oidc_issuer_endpoint]
           end
@@ -59,6 +69,10 @@ module ThreeScaleToolbox
             context[:override_private_basepath]
           end
 
+          def override_public_basepath
+            context[:override_public_basepath]
+          end
+
           def production_public_base_url
             context[:production_public_base_url]
           end
@@ -70,6 +84,30 @@ module ThreeScaleToolbox
           def override_private_base_url
             context[:override_private_base_url]
           end
+
+          def backend_api_secret_token
+            context[:backend_api_secret_token]
+          end
+
+          def backend_api_host_header
+            context[:backend_api_host_header]
+          end
+
+          def prefix_matching
+            context[:prefix_matching]
+          end
+
+          def base_path
+            api_spec.base_path || '/'
+          end
+
+          def public_base_path
+            override_public_basepath || base_path
+          end
+
+          def private_base_path
+            override_private_basepath || base_path
+          end
         end
       end
     end

data/lib/3scale_toolbox/commands/import_command/openapi/update_policies_step.rb

@@ -33,7 +33,7 @@ module ThreeScaleToolbox
 
           def add_anonymous_access_policy(policies)
             # only on 'open api' security req
-            return unless security.nil?
+            return unless api_spec.security.nil?
 
             return if policies.any? { |policy| policy['name'] == 'default_credentials' }
 
@@ -60,12 +60,12 @@ module ThreeScaleToolbox
 
           def add_rh_sso_keycloak_role_check_policy(policies)
             # only applies to oauth2 sec type
-            return if security.nil? || security.type != 'oauth2'
+            return if api_spec.security.nil? || api_spec.security[:type] != 'oauth2'
 
             return if policies.any? { |policy| policy['name'] == 'keycloak_role_check' }
 
             # only when there are scopes defined
-            return if security.scopes.empty?
+            return if api_spec.security[:scopes].empty?
 
             policies << keycloak_policy
           end
@@ -79,7 +79,7 @@ module ThreeScaleToolbox
                 scopes: [
                   {
                     realm_roles: [],
-                    client_roles: security.scopes.map { |scope| { 'name': scope } }
+                    client_roles: api_spec.security[:scopes].map { |scope| { 'name': scope } }
                   }
                 ]
               },
@@ -88,7 +88,7 @@ module ThreeScaleToolbox
           end
 
           def add_url_rewritting_policy(policies)
-            return if private_base_path == api_spec.public_base_path
+            return if private_base_path == public_base_path
 
             url_rewritting_policy_idx = policies.find_index do |policy|
               policy['name'] == 'url_rewriting'
@@ -101,10 +101,6 @@ module ThreeScaleToolbox
             end
           end
 
-          def private_base_path
-            override_private_basepath || api_spec.base_path
-          end
-
           def url_rewritting_policy
             regex = url_rewritting_policy_regex
             replace = url_rewritting_policy_replace
@@ -128,11 +124,11 @@ module ThreeScaleToolbox
           end
 
           def url_rewritting_policy_regex
-            "^#{api_spec.public_base_path}"
+            "^#{public_base_path}"
           end
 
           def url_rewritting_policy_replace
-            "#{private_base_path}"
+            private_base_path
           end
         end
       end