3scale_toolbox 0.17.1 → 0.19.0

data/lib/3scale_toolbox/entities/proxy_config.rb

@@ -52,7 +52,6 @@ module ThreeScaleToolbox
         end
         proxy_cfg
       end
-
     end
   end
 end

data/lib/3scale_toolbox/entities/service.rb

@@ -1,6 +1,8 @@
 module ThreeScaleToolbox
   module Entities
     class Service
+      include CRD::ProductSerializer
+
       VALID_PARAMS = %w[
         name backend_version deployment_option description
         system_name end_user_registration_required
@@ -29,16 +31,12 @@ module ThreeScaleToolbox
         end
 
         def find_by_system_name(remote:, system_name:)
-          service_list = remote.list_services
-
-          if service_list.respond_to?(:has_key?) && (errors = service_list['errors'])
-            raise ThreeScaleToolbox::ThreeScaleApiError.new('Service list not read', errors)
+          attrs = list_services(remote: remote).find do |svc|
+            svc['system_name'] == system_name
           end
+          return if attrs.nil?
 
-          service_attrs = service_list.find { |svc| svc['system_name'] == system_name }
-          return if service_attrs.nil?
-
-          new(id: service_attrs.fetch('id'), remote: remote, attrs: service_attrs)
+          new(id: attrs.fetch('id'), remote: remote, attrs: attrs)
         end
 
         private
@@ -61,6 +59,36 @@ module ThreeScaleToolbox
         def filtered_service_params(original_params)
           Helper.filter_params(VALID_PARAMS, original_params)
         end
+
+        def list_services(remote:)
+          services_enum(remote: remote).reduce([], :concat)
+        end
+
+        def services_enum(remote:)
+          Enumerator.new do |yielder|
+            page = 1
+            loop do
+              list = remote.list_services(
+                page: page,
+                per_page: ThreeScale::API::MAX_SERVICES_PER_PAGE
+              )
+
+              if list.respond_to?(:has_key?) && (errors = list['errors'])
+                raise ThreeScaleToolbox::ThreeScaleApiError.new('Service list not read', errors)
+              end
+
+              break if list.nil?
+
+              yielder << list
+
+              # The API response does not tell how many pages there are available
+              # If one page is not fully filled, it means that it is the last page.
+              break if list.length < ThreeScale::API::MAX_SERVICES_PER_PAGE
+
+              page += 1
+            end
+          end
+        end
       end
 
       attr_reader :id, :remote
@@ -75,6 +103,26 @@ module ThreeScaleToolbox
         @attrs ||= fetch_attrs
       end
 
+      def system_name
+        attrs['system_name']
+      end
+
+      def name
+        attrs['name']
+      end
+
+      def description
+        attrs['description']
+      end
+
+      def deployment_option
+        attrs['deployment_option']
+      end
+
+      def backend_version
+        attrs['backend_version']
+      end
+
       def update_proxy(proxy)
         new_proxy_attrs = remote.update_proxy id, proxy
 
@@ -94,37 +142,42 @@ module ThreeScaleToolbox
         proxy_attrs
       end
 
+      def cached_proxy
+        @cached_proxy ||= proxy
+      end
+
+      # @api public
+      # @return [List]
       def metrics
-        # cache result to reuse
-        metric_and_method_list = metrics_and_methods
-        hits_metric_obj = hits_metric(metric_and_method_list)
-        hits_id = hits_metric_obj.fetch('id')
+        metric_attr_list = metrics_and_methods.select { |metric_attrs| metric_attrs['parent_id'].nil? }
 
-        ThreeScaleToolbox::Helper.array_difference(metric_and_method_list, methods(hits_id)) do |metric, method|
-          ThreeScaleToolbox::Helper.compare_hashes(metric, method, %w[id])
+        metric_attr_list.map do |metric_attrs|
+          Metric.new(id: metric_attrs.fetch('id'), service: self, attrs: metric_attrs)
         end
       end
 
       def hits
-        hits_metric(metrics_and_methods)
+        metric_list = metrics_and_methods.map do |metric_attrs|
+          Metric.new(id: metric_attrs.fetch('id'), service: self, attrs: metric_attrs)
+        end
+        metric_list.find { |metric| metric.system_name == 'hits' }.tap do |hits_metric|
+          raise ThreeScaleToolbox::Error, 'missing hits metric' if hits_metric.nil?
+        end
       end
 
-      def methods(parent_metric_id)
-        service_methods = remote.list_methods id, parent_metric_id
-        if service_methods.respond_to?(:has_key?) && (errors = service_methods['errors'])
+      # @api public
+      # @return [List]
+      def methods
+        method_attr_list = remote.list_methods id, hits.id
+        if method_attr_list.respond_to?(:has_key?) && (errors = method_attr_list['errors'])
           raise ThreeScaleToolbox::ThreeScaleApiError.new('Service methods not read', errors)
         end
 
-        service_methods
-      end
-
-      def metrics_and_methods
-        m_m = remote.list_metrics id
-        if m_m.respond_to?(:has_key?) && (errors = m_m['errors'])
-          raise ThreeScaleToolbox::ThreeScaleApiError.new('Service metrics not read', errors)
+        method_attr_list.map do |method_attrs|
+          Method.new(id: method_attrs.fetch('id'),
+                     service: self,
+                     attrs: method_attrs)
         end
-
-        m_m
       end
 
       def plans
@@ -133,19 +186,24 @@ module ThreeScaleToolbox
           raise ThreeScaleToolbox::ThreeScaleApiError.new('Service plans not read', errors)
         end
 
-        service_plans
+        service_plans.map do |plan_attrs|
+          ApplicationPlan.new(id: plan_attrs.fetch('id'),
+                              service: self,
+                              attrs: plan_attrs)
+        end
       end
 
       def mapping_rules
-        remote.list_mapping_rules id
-      end
-
-      def delete_mapping_rule(rule_id)
-        remote.delete_mapping_rule(id, rule_id)
-      end
+        mr_list = remote.list_mapping_rules id
+        if mr_list.respond_to?(:has_key?) && (errors = mr_list['errors'])
+          raise ThreeScaleToolbox::ThreeScaleApiError.new('Service mapping rules not read', errors)
+        end
 
-      def create_mapping_rule(mapping_rule)
-        remote.create_mapping_rule id, mapping_rule
+        mr_list.map do |mr_attrs|
+          MappingRule.new(id: mr_attrs.fetch('id'),
+                          service: self,
+                          attrs: mr_attrs)
+        end
       end
 
       def update(svc_attrs)
@@ -165,7 +223,12 @@ module ThreeScaleToolbox
       end
 
       def policies
-        remote.show_policies id
+        policy_chain = remote.show_policies id
+        if policy_chain.respond_to?(:has_key?) && (errors = policy_chain['errors'])
+          raise ThreeScaleToolbox::ThreeScaleApiError.new('Service policies not read', errors)
+        end
+
+        policy_chain
       end
 
       def update_policies(params)
@@ -179,8 +242,12 @@ module ThreeScaleToolbox
           raise ThreeScaleToolbox::ThreeScaleApiError.new('Service activedocs not read', errors)
         end
 
-        tenant_activedocs.select do |activedoc|
-          activedoc['service_id'] == id
+        service_activedocs = tenant_activedocs.select do |activedoc_attrs|
+          activedoc_attrs['service_id'] == id
+        end
+
+        service_activedocs.map do |activedoc_attrs|
+          Entities::ActiveDocs.new(id: activedoc_attrs.fetch('id'), remote: remote, attrs: activedoc_attrs)
         end
       end
 
@@ -194,6 +261,10 @@ module ThreeScaleToolbox
         service_oidc
       end
 
+      def cached_oidc
+        @cached_oidc ||= oidc
+      end
+
       def update_oidc(oidc_settings)
         new_oidc = remote.update_oidc(id, oidc_settings)
 
@@ -262,19 +333,42 @@ module ThreeScaleToolbox
         end
       end
 
-      def ==(other)
-        remote.http_client.endpoint == other.remote.http_client.endpoint && id == other.id
+      def create_mapping_rule(mr_attrs)
+        Entities::MappingRule.create(service: self, attrs: mr_attrs)
       end
 
-      private
+      def proxy_deploy
+        proxy_attrs = remote.proxy_deploy id
+        if proxy_attrs.respond_to?(:has_key?) && (errors = proxy_attrs['errors'])
+          raise ThreeScaleToolbox::ThreeScaleApiError.new('Proxy configuration not deployed', errors)
+        end
+
+        proxy_attrs
+      end
+
+      # Compute matrics mapping between products, including related backend metrics as well
+      def metrics_mapping(other)
+        mapping = (metrics + methods).product(other.metrics + other.methods).select do |m_a, m_b|
+          m_a.system_name == m_b.system_name
+        end.map { |m_a, m_b| [m_a.id, m_b.id] }.to_h
 
-      def hits_metric(metric_list)
-        hits_metric = metric_list.find { |metric| metric['system_name'] == 'hits' }
-        raise ThreeScaleToolbox::Error, 'missing hits metric' if hits_metric.nil?
+        backend_pairs = backend_usage_list.map(&:backend).product(other.backend_usage_list.map(&:backend)).select do |b_a, b_b|
+          b_a.system_name == b_b.system_name
+        end
+
+        backend_pairs.each do |b_a, b_b|
+          mapping.merge!(b_a.metrics_mapping(b_b))
+        end
 
-        hits_metric
+        mapping
       end
 
+      def ==(other)
+        remote.http_client.endpoint == other.remote.http_client.endpoint && id == other.id
+      end
+
+      private
+
       def fetch_attrs
         raise ThreeScaleToolbox::InvalidIdError if id.zero?
 
@@ -300,6 +394,15 @@ module ThreeScaleToolbox
 
         new_attrs
       end
+
+      def metrics_and_methods
+        m_m = remote.list_metrics id
+        if m_m.respond_to?(:has_key?) && (errors = m_m['errors'])
+          raise ThreeScaleToolbox::ThreeScaleApiError.new('Service metrics not read', errors)
+        end
+
+        m_m
+      end
     end
   end
 end

data/lib/3scale_toolbox/helper.rb

@@ -51,24 +51,21 @@ module ThreeScaleToolbox
         Array(Hash(error)['period']).any? { |msg| msg.match(/has already been taken/) }
       end
 
-      def metrics_mapping(source_metrics, target_metrics)
-        target_metrics.map do |target|
-          source = source_metrics.find do |m|
-            compare_hashes(m, target, ['system_name'])
-          end || {}
-
-          [source['id'], target['id']]
-        end.to_h
+      def application_plan_mapping(source_app_plans, target_app_plans)
+        target_app_plans.map do |target|
+          source = source_app_plans.find { |app_plan| app_plan.system_name == target.system_name }
+          [source, target]
+        end.reject { |key, _| key.nil? }
       end
 
-      def application_plan_mapping(source_app_plans, target_app_plans)
-        mapping = target_app_plans.map do |target|
-          source = source_app_plans.find do |app_plan|
-            compare_hashes(app_plan, target, ['system_name'])
-          end || {}
-          [source['id'], target]
-        end
-        mapping.reject { |key, _| key.nil? }
+      def random_lowercase_name(length=8)
+        [*('a'..'z')].sample(length).join
+      end
+
+      def backend_metric_link_parser(link)
+        return if link.nil?
+
+        link.match(/admin\/api\/backend_apis\/(\d+)\/metrics/) { |m| m.captures[0] }
       end
     end
 

data/lib/3scale_toolbox/proxy_logger.rb

@@ -13,6 +13,10 @@ module ThreeScaleToolbox
       result
     end
 
+    def public_send(name, *args)
+      method_missing(name, *args)
+    end
+
     def respond_to_missing?(method_name, include_private = false)
       super
     end

data/lib/3scale_toolbox/remote_cache.rb

@@ -0,0 +1,164 @@
+module ThreeScaleToolbox
+  class RemoteCache < BasicObject
+
+    attr_reader :metrics_cache, :methods_cache, :backend_metrics_cache, :backend_methods_cache, :subject
+
+    def initialize(subject)
+      @subject = subject
+      # Metrics and methods cache data
+      @metrics_cache = {}
+      # methods cache data
+      @methods_cache = {}
+      # Backend Metrics and methods cache data
+      @backend_metrics_cache = {}
+      # Backend methods cache data
+      @backend_methods_cache = {}
+    end
+
+    def list_metrics(service_id)
+      return metrics_cache[service_id] if metrics_cache.has_key? service_id
+
+      subject.list_metrics(service_id).tap do |metrics|
+        metrics_cache[service_id] = metrics unless metrics.respond_to?(:has_key?) && !metrics['errors'].nil?
+      end
+    end
+
+    def list_methods(service_id, metric_id)
+      key = method_cache_key(service_id, metric_id)
+      return methods_cache[key] if methods_cache.has_key? key
+
+      subject.list_methods(service_id, metric_id).tap do |methods|
+        methods_cache[key] = methods unless methods.respond_to?(:has_key?) && !methods['errors'].nil?
+      end
+    end
+
+    def create_metric(service_id, attributes)
+      subject.create_metric(service_id, attributes).tap do |metric_attrs|
+        metrics_cache.delete(service_id) unless metric_attrs.respond_to?(:has_key?) && !metric_attrs['errors'].nil?
+      end
+    end
+
+    def update_metric(service_id, metric_id, attributes)
+      subject.update_metric(service_id, metric_id, attributes).tap do |metric_attrs|
+        metrics_cache.delete(service_id) unless metric_attrs.respond_to?(:has_key?) && !metric_attrs['errors'].nil?
+      end
+    end
+
+    def delete_metric(service_id, metric_id)
+      subject.delete_metric(service_id, metric_id).tap do |_|
+        metrics_cache.delete(service_id)
+      end
+    end
+
+    def update_method(service_id, parent_id, id, attributes)
+      subject.update_method(service_id, parent_id, id, attributes).tap do |method_attrs|
+        metrics_cache.delete(service_id) unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+        methods_cache.delete(method_cache_key(service_id, parent_id)) unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+      end
+    end
+
+
+    def create_method(service_id, metric_id, attributes)
+      subject.create_method(service_id, metric_id, attributes).tap do |method_attrs|
+        metrics_cache.delete(service_id) unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+        methods_cache.delete(method_cache_key(service_id, metric_id)) unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+      end
+    end
+
+    def delete_method(service_id, parent_id, id)
+      subject.delete_method(service_id, parent_id, id).tap do |_|
+        metrics_cache.delete(service_id)
+        methods_cache.delete(method_cache_key(service_id, parent_id))
+      end
+    end
+
+    ###
+    # Backends
+    ###
+
+    def list_backend_metrics(backend_id)
+      return backend_metrics_cache[backend_id] if backend_metrics_cache.has_key? backend_id
+
+      subject.list_backend_metrics(backend_id).tap do |metrics|
+        backend_metrics_cache[backend_id] = metrics unless metrics.respond_to?(:has_key?) && !metrics['errors'].nil?
+      end
+    end
+
+    def list_backend_methods(backend_id, metric_id)
+      key = method_cache_key(backend_id, metric_id)
+      return backend_methods_cache[key] if backend_methods_cache.has_key? key
+
+      subject.list_backend_methods(backend_id, metric_id).tap do |methods|
+        backend_methods_cache[key] = methods unless methods.respond_to?(:has_key?) && !methods['errors'].nil?
+      end
+    end
+
+    def create_backend_metric(backend_id, attributes)
+      subject.create_backend_metric(backend_id, attributes).tap do |metric_attrs|
+        backend_metrics_cache.delete(backend_id) unless metric_attrs.respond_to?(:has_key?) && !metric_attrs['errors'].nil?
+      end
+    end
+
+    def update_backend_metric(backend_id, metric_id, attributes)
+      subject.update_backend_metric(backend_id, metric_id, attributes).tap do |metric_attrs|
+        backend_metrics_cache.delete(backend_id) unless metric_attrs.respond_to?(:has_key?) && !metric_attrs['errors'].nil?
+      end
+    end
+
+    def delete_backend_metric(backend_id, metric_id)
+      subject.delete_backend_metric(backend_id, metric_id).tap do |_|
+        backend_metrics_cache.delete(backend_id)
+      end
+    end
+
+    def create_backend_method(backend_id, metric_id, attributes)
+      subject.create_backend_method(backend_id, metric_id, attributes).tap do |method_attrs|
+        unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+          backend_metrics_cache.delete(backend_id)
+          backend_methods_cache.delete(method_cache_key(backend_id, metric_id))
+        end
+      end
+    end
+
+    def delete_backend_method(backend_id, metric_id, method_id)
+      subject.delete_backend_method(backend_id, metric_id, method_id).tap do |_|
+        backend_metrics_cache.delete(backend_id)
+        backend_methods_cache.delete(method_cache_key(backend_id, metric_id))
+      end
+    end
+
+    def update_backend_method(backend_id, metric_id, method_id, attributes)
+      subject.update_backend_method(backend_id, metric_id, method_id, attributes).tap do |method_attrs|
+        unless method_attrs.respond_to?(:has_key?) && !method_attrs['errors'].nil?
+          backend_metrics_cache.delete(backend_id)
+          backend_methods_cache.delete(method_cache_key(backend_id, metric_id))
+        end
+      end
+    end
+
+    ###
+    # Generic methods
+    ###
+
+    def method_missing(name, *args, **kwargs)
+      # Only needed to keep ruby 2.6 support
+      # For ruby >= 2.7 it is enough with
+      # subject.public_send(name, *args, **kwargs) 
+      if kwargs.empty?
+        subject.public_send(name, *args)
+      else
+        subject.public_send(name, *args, **kwargs)
+      end
+    end
+
+    def respond_to_missing?(method_name, include_private = false)
+      super
+    end
+
+    private
+
+    def method_cache_key(id, metric_id)
+      "#{id}#{metric_id}"
+    end
+  end
+end