apisonator 2.100.0

data/lib/3scale/backend/memoizer.rb

@@ -0,0 +1,277 @@
+raise "Memoizer is not thread safe" if ThreeScale::Backend::Manifest.thread_safe?
+
+module ThreeScale
+  module Backend
+    class Memoizer
+      EXPIRE = 60
+      PURGE = 60
+      MAX_ENTRIES = 10000
+      ACTIVE = true
+      private_constant :EXPIRE, :PURGE, :MAX_ENTRIES, :ACTIVE
+
+      def self.reset!
+        # Initialize class instance variables
+        # Note: we would be better off pre-allocating the Hash size
+        # ie. rb_hash_new_with_size(MAX_ENTRIES)
+        @memoizer_cache = Hash.new
+        @memoizer_purge_time = Time.now.getutc.to_i + PURGE
+        @memoizer_stats_count = 0
+        @memoizer_stats_hits = 0
+      end
+
+      reset!
+
+      # Key management:
+      #
+      # When automatically memoizing using the decorator, you should
+      # NEVER assume a specific key format. Here are some helpers to
+      # let you build keys to clear or getting them.
+      #
+      # You should only use build_keys_for_class and build_key
+      #
+
+      private
+
+      def self.build_class_key(klass)
+        classkey = klass.to_s
+        # This can receive both objects and classes, so check if we can
+        # call singleton_class? before actually doing so.
+        if klass.respond_to? :singleton_class? and klass.singleton_class?
+          # obtain class from Ruby's metaclass notation
+          classkey.split(':'.freeze).delete_if do |k|
+            k[0] == '#'.freeze
+          end.join(':'.freeze).split('>'.freeze).first
+        else
+          classkey
+        end
+      end
+
+      def self.build_method_key(classkey, methodname)
+        classkey + '.'.freeze + methodname
+      end
+
+      def self.build_args_key(methodkey, *args)
+        if args.empty?
+          methodkey
+        else
+          methodkey + '-'.freeze + args.join('-'.freeze)
+        end
+      end
+
+      # A method to inspect in debugging mode the contents of the cache
+      def self.cache
+        raise 'Memoizer.cache only in development!' unless ThreeScale::Backend.development?
+        @memoizer_cache
+      end
+
+      public
+
+      # Generate a key for the given class, method and args
+      def self.build_key(klass, method, *args)
+        key = build_class_key klass
+        key = build_method_key key, method.to_s
+        build_args_key key, *args
+      end
+
+      # Pass in the class or object that receives the memoized
+      # methods, and a hash containing the methods as keys and
+      # an array of their arguments in order.
+      def self.build_keys_for_class(klass, methods_n_args)
+        classkey = build_class_key klass
+        methods_n_args.map do |method, args|
+          key = build_method_key(classkey, method.to_s)
+          build_args_key key, *args
+        end
+      end
+
+      if ACTIVE
+        Entry = Struct.new(:obj, :expire)
+        private_constant :Entry
+
+        def self.fetch(key)
+          @memoizer_stats_count = @memoizer_stats_count + 1
+
+          cached = @memoizer_cache[key]
+
+          now = Time.now.getutc.to_i
+          purge(now) if now > @memoizer_purge_time
+
+          if cached && now <= cached.expire
+            @memoizer_stats_hits = @memoizer_stats_hits + 1
+            cached
+          end
+        end
+
+        def self.memoize(key, obj)
+          @memoizer_cache[key] = Entry.new(obj, Time.now.getutc.to_i + EXPIRE)
+          obj
+        end
+      else
+        def self.fetch(_key)
+          nil
+        end
+
+        def self.memoize(_key, obj)
+          obj
+        end
+      end
+
+      def self.get(key)
+        entry = @memoizer_cache[key]
+        entry.obj if entry
+      end
+
+      def self.memoized?(key)
+        !!(fetch key)
+      end
+
+      def self.clear(keys)
+        Array(keys).each do |key|
+          @memoizer_cache.delete key
+        end
+      end
+
+      def self.purge(time)
+        @memoizer_purge_time = time + PURGE
+
+        @memoizer_cache.delete_if do |_, entry|
+          time > entry.expire
+        end
+
+        ##safety, should never reach this unless massive concurrency
+        reset! if @memoizer_cache.size > MAX_ENTRIES
+      end
+
+      def self.stats
+        {
+          size: @memoizer_cache.size,
+          count: @memoizer_stats_count,
+          hits: @memoizer_stats_hits,
+        }
+      end
+
+      def self.memoize_block(key, &block)
+        entry = fetch key
+        if entry.nil?
+          Memoizer.memoize(key, yield)
+        else
+          entry.obj
+        end
+      end
+
+      # Decorator allows a class or module to include it and get
+      #   memoize :method1, :method2, ...
+      # using keys "#{classname}.#{methodname}-#{arg1}-#{arg2}-..."
+      module Decorator
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          module Helpers
+            module_function
+
+            def memoize_instance_method(m, klass)
+              method_name = m.name
+              method_s = method_name.to_s
+              klass.send :define_method, method_name do |*args|
+                key = Memoizer.build_method_key self.to_s, method_s
+                key = Memoizer.build_args_key key, *args
+                Memoizer.memoize_block(key) do
+                  m.bind(self).call(*args)
+                end
+              end
+            end
+            private :memoize_instance_method
+
+            def memoize_class_method(m, partialkey, klass)
+              klass.define_singleton_method(m.name) do |*args|
+                key = Memoizer.build_args_key partialkey, *args
+                Memoizer.memoize_block(key) do
+                  m.call(*args)
+                end
+              end
+            end
+            private :memoize_class_method
+
+            # helper to go down one level from the current class context
+            # ie. the reverse of singleton_class: from metaclass to class
+            def get_instance_class(klass)
+              return klass unless klass.singleton_class?
+              # workaround Ruby's lack of the inverse of singleton_class
+              base_s = klass.to_s.split(':').delete_if { |k| k.start_with? '#' }.
+                join(':').split('>').first
+              iklass = Kernel.const_get(base_s)
+              # got the root class, now go up a level shy of self
+              iklass = iklass.singleton_class while iklass.singleton_class != klass
+              iklass
+            end
+            private :get_instance_class
+          end
+
+          # memoize :method, :other_method, ...
+          #
+          # Decorate the methods passed in memoizing their results based
+          # on the parameters they receive using a key with the form:
+          # (ClassName|Instance).methodname[-param1[-param2[-...]]]
+          #
+          # You can call this from a class on instance or class methods, and
+          # from a metaclass on instance methods, which actually are class
+          # methods.
+          #
+          # CAVEAT: if you have an instance method named exactly as an existing
+          # class method you either memoize the instance method BEFORE defining
+          # the class method or use memoize_i on the instance method.
+          #
+          # WARNING: do NOT use this memoize method on frequently called instance
+          # methods since you'll have a noticeable overhead from the necessity
+          # to bind the method to the object.
+          #
+          def memoize(*methods)
+            classkey = Memoizer.build_class_key self
+            # get the base class of self so that we get rid of metaclasses
+            klass = Helpers.get_instance_class self
+            # make sure klass points to the klass that self is a metaclass of
+            # in case we're being invoked from a metaclass
+            methods.each do |m|
+              # For each method, first search for a class method, which is the
+              # common case. If not found, then look for an instance method.
+              #
+              begin
+                key = Memoizer.build_method_key(classkey, m.to_s)
+                original_method = klass.method m
+                raise NameError unless original_method.owner == klass.singleton_class
+                Helpers.memoize_class_method original_method, key, klass
+              rescue NameError
+                # If we cannot find a class method, try an instance method
+                # before bailing out.
+                memoize_i m
+              end
+            end
+          end
+
+          # memoize_i :method, :other_method
+          #
+          # Forces memoization of methods which are instance methods in
+          # the current context level. This can be used to, for example,
+          # override the default look up when we have two methods, class
+          # and instance, which are named the same and we want to memoize
+          # both or only the instance level one.
+          def memoize_i(*methods)
+            methods.each do |m|
+              # We don't support calling this from a metaclass, because
+              # we have not built a correct key. The user should use memoize
+              # instead of this, which will already work with the instance
+              # methods within a metaclass, that is, the class' class methods.
+              raise NameError if singleton_class?
+              original_method = instance_method m
+              raise NameError unless original_method.owner == self
+              Helpers.memoize_instance_method original_method, self
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/3scale/backend/metric.rb

@@ -0,0 +1,275 @@
+require 'set'
+
+module ThreeScale
+  module Backend
+    class Metric
+      module KeyHelpers
+        def key(service_id, id, attribute)
+          encode_key("metric/service_id:#{service_id}/id:#{id}/#{attribute}")
+        end
+
+        def id_key(service_id, name)
+          encode_key("metric/service_id:#{service_id}/name:#{name}/id")
+        end
+
+        def id_set_key(service_id)
+          encode_key("metrics/service_id:#{service_id}/ids")
+        end
+      end
+
+      include KeyHelpers
+      extend KeyHelpers
+
+      include Storable
+
+      attr_accessor :service_id, :id, :parent_id, :name
+      attr_writer :children
+
+      def save
+        old_name = self.class.load_name(service_id, id)
+        storage.pipelined do
+          save_attributes
+          save_to_list
+          remove_reverse_mapping(service_id, old_name) if old_name != name
+        end
+
+        # can't include this in the pipeline since it is a potentially
+        # large number of commands.
+        save_children
+
+        self.class.clear_cache(service_id, id, name)
+      end
+
+      def children
+        @children ||= []
+      end
+
+      def to_hash
+        {
+          service_id: service_id,
+          id: id,
+          parent_id: parent_id,
+          name: name
+        }
+      end
+
+      def update(attributes)
+        attributes.each do |attr, val|
+          public_send("#{attr}=", val)
+        end
+        self
+      end
+
+      class << self
+        include Memoizer::Decorator
+
+        def attribute_names
+          %i[service_id id parent_id name children].freeze
+        end
+
+        def load(service_id, id)
+          name, parent_id = storage.mget(key(service_id, id, :name),
+                                         key(service_id, id, :parent_id))
+
+          name && new(id: id.to_s,
+                      service_id: service_id.to_s,
+                      name: name,
+                      parent_id: parent_id)
+        end
+
+        def load_all(service_id)
+          Collection.new(service_id)
+        end
+        memoize :load_all
+
+        def load_id(service_id, name)
+          storage.get(id_key(service_id, name))
+        end
+        memoize :load_id
+
+        def load_all_ids(service_id)
+          # smembers is guaranteed to return an array of strings, even if empty
+          storage.smembers(id_set_key(service_id))
+        end
+        memoize :load_all_ids
+
+        def load_name(service_id, id)
+          storage.get(key(service_id, id, :name))
+        end
+        memoize :load_name
+
+        def load_all_names(service_id, ids)
+          if ids.nil? || ids.empty?
+            {}
+          else
+            name_keys = ids.map { |id| key(service_id, id, :name) }
+            Hash[ids.zip(storage.mget(name_keys))]
+          end
+        end
+        memoize :load_all_names
+
+        def load_parent_id(service_id, id)
+          storage.get(key(service_id, id, :parent_id))
+        end
+        memoize :load_parent_id
+
+        def save(attributes)
+          metrics = new(attributes)
+          metrics.save
+          metrics
+        end
+
+        # Returns a hash where the keys can only be parent metric ids (as
+        # Strings) and their values are arrays of children.
+        #
+        # The as_names optional parameter (default: true) returns metric names
+        # instead of metric ids when true.
+        def hierarchy(service_id, as_names = true)
+          h_ids = hierarchy_ids(service_id)
+          return h_ids unless as_names
+
+          metric_ids = Set.new(h_ids.keys + h_ids.values.flatten)
+          return {} if metric_ids.empty?
+
+          res = {}
+          metric_names = load_all_names(service_id, metric_ids)
+
+          h_ids.each do |m_id, c_ids|
+            m_name = metric_names[m_id]
+            res[m_name] = c_ids.map do |c_id|
+              metric_names[c_id]
+            end
+          end
+
+          res
+        end
+        memoize :hierarchy
+
+        def children(service_id, id)
+          hierarchy(service_id, false)[id.to_s]
+        end
+
+        # Returns the "descendants" of a metric, that is, its children,
+        # grandchildren, etc. in the metric hierarchy of the given service.
+        # In other words, the "descendants" of a metric are its children plus
+        # the descendants of each of them.
+        def descendants(service_id, metric_name)
+          metrics_hierarchy = hierarchy(service_id)
+          children = metrics_hierarchy[metric_name] || []
+
+          children.reduce(children) do |acc, child|
+            acc + descendants(service_id, child)
+          end
+        end
+        memoize :descendants
+
+        # Returns the "ascendants" of a metric, that is, its parent,
+        # grandparent, etc. The "ascendants" of a metric are its parent plus
+        # the ascendants of the parent.
+        def ascendants(service_id, metric_name)
+          parents_of_metric = parents(service_id, [metric_name])
+
+          parents_of_metric.reduce(parents_of_metric) do |acc, parent|
+            acc + ascendants(service_id, parent)
+          end
+        end
+        memoize :ascendants
+
+        # Given an array of metrics, returns an array without duplicates that
+        # includes the names of the metrics that are parent of at least one of
+        # the given metrics.
+        def parents(service_id, metric_names)
+          parents = []
+
+          metric_names.each do |name|
+            parent_id = load_parent_id service_id, load_id(service_id, name)
+            if parent_id
+              parents << load_name(service_id, parent_id)
+            end
+          end
+
+          parents.uniq
+        end
+
+        def delete(service_id, id)
+          name = load_name(service_id, id)
+          return false unless name and not name.empty?
+          clear_cache(service_id, id, name)
+
+          storage.pipelined do
+            storage.srem(id_set_key(service_id), id)
+
+            storage.del(key(service_id, id, :name),
+                        key(service_id, id, :parent_id),
+                        id_key(service_id, name))
+          end
+
+          true
+        end
+
+        def clear_cache(service_id, id, name)
+          metric_ids = load_all_ids(service_id)
+          Memoizer.clear(Memoizer.build_keys_for_class(self,
+                                        load_all: [service_id],
+                                        load_all_names: [service_id, metric_ids],
+                                        load_name: [service_id, id],
+                                        load_id: [service_id, name],
+                                        load_all_ids: [service_id]))
+        end
+
+        private
+
+        def hierarchy_ids(service_id)
+          ids = load_all_ids(service_id)
+          parent_ids_keys = ids.map { |id| key(service_id, id, :parent_id) }
+
+          parent_ids = storage.pipelined do
+            parent_ids_keys.each_slice(PIPELINED_SLICE_SIZE).map do |slice|
+              storage.mget(slice)
+            end
+          end.flatten
+
+          parent_child_rels = parent_ids.zip(ids)
+
+          parent_child_rels.inject({}) do |acc, (parent_id, child_id)|
+            if parent_id # nil if child_id has no parent
+              acc[parent_id] ||= []
+              acc[parent_id] << child_id
+            end
+            acc
+          end
+        end
+      end
+
+      private
+
+      def remove_reverse_mapping(service_id, name)
+        storage.del id_key(service_id, name)
+      end
+
+      def save_attributes
+        storage.set(id_key(service_id, name), id)
+        storage.set(key(service_id, id, :name), name)
+        storage.set(key(service_id, id, :parent_id), parent_id) if parent_id
+      end
+
+      def save_to_list
+        storage.sadd(id_set_key(service_id), id)
+      end
+
+      def save_children
+        children.each do |child|
+          child.service_id = service_id
+          child.parent_id = id
+          child.save
+        end
+      end
+
+    end
+  end
+end
+
+# this is required by our code, but it is nested inside class Metric
+# require'ing it here ensures we always reopen the class instead of
+# defining it.
+require '3scale/backend/metric/collection'