zermelo 1.0.1 → 1.1.0

data/lib/zermelo/filters/steps/set_step.rb

@@ -0,0 +1,176 @@
+require 'zermelo/filters/steps/base_step'
+
+# NB: temp keys for now are bare redis keys, should be full Key objects
+module Zermelo
+  module Filters
+    class Steps
+      class SetStep < Zermelo::Filters::Steps::BaseStep
+        def self.accepted_types
+          [:set, :sorted_set] # TODO should allow :list as well
+        end
+
+        def self.returns_type
+          :set
+        end
+
+        REDIS_SHORTCUTS = {
+          :ids     => proc {|key|     Zermelo.redis.smembers(key) },
+          :count   => proc {|key|     Zermelo.redis.scard(key) },
+          :exists? => proc {|key, id| Zermelo.redis.sismember(key, id) }
+        }
+
+        def resolve(backend, associated_class, opts = {})
+
+          case backend
+          when Zermelo::Backends::RedisBackend
+            source     = opts[:source]
+            idx_attrs  = opts[:index_attrs]
+            attr_types = opts[:attr_types]
+            temp_keys  = opts[:temp_keys]
+
+            source_keys = @attributes.inject([]) do |memo, (att, value)|
+
+              val = value.is_a?(Set) ? value.to_a : value
+
+              if :id.eql?(att)
+                ts = associated_class.send(:temp_key, :set)
+                temp_keys << ts
+                Zermelo.redis.sadd(backend.key_to_redis_key(ts), val)
+                memo << ts
+              else
+                idx_class = idx_attrs[att.to_s]
+                raise "'#{att}' property is not indexed" if idx_class.nil?
+
+                if val.is_a?(Enumerable)
+                  conditions_set = associated_class.send(:temp_key, :set)
+                  r_conditions_set = backend.key_to_redis_key(conditions_set)
+
+                  backend.temp_key_wrap do |conditions_temp_keys|
+                    index_keys = val.collect {|v|
+                      il = backend.index_lookup(att, associated_class,
+                        idx_class, v, attr_types[att], conditions_temp_keys)
+                      backend.key_to_redis_key(il)
+                    }
+
+                    Zermelo.redis.sunionstore(r_conditions_set, *index_keys)
+                  end
+                  memo << conditions_set
+                else
+                  memo << backend.index_lookup(att, associated_class,
+                            idx_class, val, attr_types[att], temp_keys)
+                end
+              end
+
+              memo
+            end
+
+            case source.type
+            when :sorted_set
+              Zermelo::Filters::Steps::SortedSetStep.evaluate(backend,
+                @options[:op], associated_class, source, source_keys, temp_keys, opts)
+            when :set
+              self.class.evaluate(backend, @options[:op], associated_class,
+                source, source_keys, temp_keys, opts)
+            end
+
+          when Zermelo::Backends::InfluxDBBackend
+            query = ''
+
+            unless opts[:first].is_a?(TrueClass)
+              case @options[:op]
+              when :intersect, :diff
+                query += ' AND '
+              when :union
+                query += ' OR '
+              end
+            end
+
+            case @options[:op]
+            when :intersect, :union
+              query += @attributes.collect {|k, v|
+                op, value = case v
+                when String
+                  ["=~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+                else
+                  ["=",  "'#{v}'"]
+                end
+
+               "#{k} #{op} #{value}"
+              }.join(' AND ')
+
+            when :diff
+              query += @attributes.collect {|k, v|
+                op, value = case v
+                when String
+                  ["!~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+                else
+                  ["!=",  "'#{v}'"]
+                end
+
+                "#{k} #{op} #{value}"
+              }.join(' AND ')
+            else
+              raise "Unhandled filter operation '#{@options[:op]}'"
+            end
+
+            query += ")"
+
+            query
+          end
+        end
+
+        def self.evaluate(backend, op, associated_class, source, source_keys, temp_keys, opts = {})
+          shortcut = opts[:shortcut]
+
+          r_source_key  = backend.key_to_redis_key(source)
+          r_source_keys = source_keys.collect {|sk| backend.key_to_redis_key(sk) }
+
+          if :ids.eql?(shortcut)
+            case op
+            when :union
+              backend.temp_key_wrap do |shortcut_temp_keys|
+                dest_set = associated_class.send(:temp_key, :set)
+                shortcut_temp_keys << dest_set
+                r_dest_set = backend.key_to_redis_key(dest_set)
+
+                Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+                Zermelo.redis.sunion(r_dest_set, r_source_key)
+              end
+            when :intersect
+              Zermelo.redis.sinter(r_source_key, *r_source_keys)
+            when :diff
+              backend.temp_key_wrap do |shortcut_temp_keys|
+                dest_set = associated_class.send(:temp_key, :set)
+                shortcut_temp_keys << dest_set
+                r_dest_set = backend.key_to_redis_key(dest_set)
+
+                Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+                Zermelo.redis.sdiff(r_source_key, r_dest_set)
+              end
+            end
+          else
+            dest_set = associated_class.send(:temp_key, :set)
+            r_dest_set = backend.key_to_redis_key(dest_set)
+            temp_keys << dest_set
+
+            case op
+            when :union
+              Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+              Zermelo.redis.sunionstore(r_dest_set, r_source_key, r_dest_set)
+            when :intersect
+              Zermelo.redis.sinterstore(r_dest_set, r_source_key, *r_source_keys)
+            when :diff
+              Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+              Zermelo.redis.sdiffstore(r_dest_set, r_source_key, r_dest_set)
+            end
+
+            return dest_set if shortcut.nil?
+            REDIS_SHORTCUTS[shortcut].call(*([r_dest_set] + opts[:shortcut_args]))
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/zermelo/filters/steps/sort_step.rb

@@ -11,6 +11,91 @@ module Zermelo
         def self.returns_type
           :list
         end
+
+        def resolve(backend, associated_class, opts = {})
+          case backend
+          when Zermelo::Backends::RedisBackend
+            source = opts[:source]
+            idx_attrs = opts[:index_attrs]
+            attr_types = opts[:attr_types]
+            temp_keys = opts[:temp_keys]
+
+            dest_list = associated_class.send(:temp_key, :list)
+            temp_keys << dest_list
+            r_dest_list = backend.key_to_redis_key(dest_list)
+
+            # TODO raise error in step construction if keys not
+            # passed as expected below
+            sort_attrs_and_orders = case options[:keys]
+            when String, Symbol
+              {options[:keys].to_s => options[:desc].is_a?(TrueClass) ? :desc : :asc}
+            when Array
+              options[:keys].each_with_object({}) do |k, memo|
+                memo[k.to_sym] = (options[:desc].is_a?(TrueClass) ? :desc : :asc)
+              end
+            when Hash
+              options[:keys]
+            end
+
+            # TODO check if complex attribute types or associations
+            # can be used for sorting
+
+            r_source = backend.key_to_redis_key(source)
+
+            # this set will be overwritten by the result list
+            case source.type
+            when :set
+              Zermelo.redis.sunionstore(r_dest_list, r_source)
+            when :sorted_set
+              Zermelo.redis.zunionstore(r_dest_list, [r_source])
+            end
+
+            class_key = associated_class.send(:class_key)
+
+            sort_attrs_and_orders.keys.reverse.each_with_index do |sort_attr, idx|
+
+              order = sort_attrs_and_orders[sort_attr]
+
+              sort_opts = {}
+
+              unless 'id'.eql?(sort_attr.to_s)
+                sort_opts.update(:by => "#{class_key}:*:attrs->#{sort_attr}")
+              end
+
+              if (idx + 1) == sort_attrs_and_orders.size
+                # only apply offset & limit on the last sort
+                o = options[:offset]
+                l = options[:limit]
+
+                if !(l.nil? && o.nil?)
+                  o = o.nil? ? 0 : o.to_i
+                  l = (l.nil? || (l.to_i < 1)) ? (Zermelo.redis.llen(dest_list) - o) : l
+                  sort_opts.update(:limit => [o, l])
+                end
+              end
+
+              order_parts = []
+              sort_attr_type = attr_types[sort_attr.to_sym]
+              unless [:integer, :float, :timestamp].include?(sort_attr_type)
+                order_parts << 'alpha'
+              end
+              order_parts << 'desc' if 'desc'.eql?(order.to_s)
+
+              unless order_parts.empty?
+                sort_opts.update(:order => order_parts.join(' '))
+              end
+
+              sort_opts.update(:store => r_dest_list)
+              Zermelo.redis.sort(r_dest_list, sort_opts)
+            end
+
+            shortcut = opts[:shortcut]
+
+            return dest_list if shortcut.nil?
+            Zermelo::Filters::Steps::ListStep::REDIS_SHORTCUTS[shortcut].
+              call(*([r_dest_list] + opts[:shortcut_args]))
+          end
+        end
       end
     end
   end

data/lib/zermelo/filters/steps/sorted_set_step.rb

@@ -0,0 +1,156 @@
+require 'zermelo/filters/steps/base_step'
+
+module Zermelo
+  module Filters
+    class Steps
+      class SortedSetStep < Zermelo::Filters::Steps::BaseStep
+        def self.accepted_types
+          [:sorted_set]
+        end
+
+        def self.returns_type
+          :sorted_set
+        end
+
+        REDIS_SHORTCUTS = {
+          :ids     => proc {|key|     Zermelo.redis.zrange(key, 0, -1) },
+          :count   => proc {|key|     Zermelo.redis.zcard(key) },
+          :exists? => proc {|key, id| !Zermelo.redis.zscore(key, id).nil? },
+          :first   => proc {|key|     Zermelo.redis.zrange(key, 0, 0).first },
+          :last    => proc {|key|     Zermelo.redis.zrevrange(key, 0, 0).first }
+        }
+
+        def resolve(backend, associated_class, opts = {})
+          op     = @options[:op]
+          start  = @options[:start]
+          finish = @options[:finish]
+
+          case backend
+          when Zermelo::Backends::RedisBackend
+            source = opts[:source]
+            idx_attrs = opts[:index_attrs]
+            attr_types = opts[:attr_types]
+            temp_keys = opts[:temp_keys]
+
+            range_temp_key = associated_class.send(:temp_key, :sorted_set)
+            temp_keys << range_temp_key
+            range_ids_set = backend.key_to_redis_key(range_temp_key)
+
+            if @options[:by_score]
+              start = '-inf' if start.nil? || (start <= 0)
+              finish = '+inf' if finish.nil? || (finish <= 0)
+            else
+              start = 0 if start.nil?
+              finish = -1 if finish.nil?
+            end
+
+            args = [start, finish]
+
+            if @options[:by_score]
+              query = :zrangebyscore
+              args = args.map(&:to_s)
+            else
+              query = :zrange
+            end
+
+            args << {:with_scores => :true}
+
+            if @options[:limit]
+              args.last.update(:limit => [0, @options[:limit].to_i])
+            end
+
+            r_source   = backend.key_to_redis_key(source)
+            args.unshift(r_source)
+
+            range_ids_scores = Zermelo.redis.send(query, *args)
+
+            unless range_ids_scores.empty?
+              Zermelo.redis.zadd(range_ids_set, range_ids_scores.map(&:reverse))
+            end
+
+            self.class.evaluate(backend, @options[:op], associated_class,
+              source, [range_temp_key], temp_keys, opts)
+
+          when Zermelo::Backends::InfluxDBBackend
+
+            query = ''
+
+            unless opts[:first].is_a?(TrueClass)
+              case @options[:op]
+              when :intersect_range, :diff_range
+                query += ' AND '
+              when :union_range
+                query += ' OR '
+              end
+            end
+
+            start  = nil if !start.nil?  && (start <= 0)
+            finish = nil if !finish.nil? && (finish <= 0)
+
+            unless start.nil? && finish.nil?
+              time_q = []
+
+              case @options[:op]
+              when :intersect_range, :union_range
+                unless start.nil?
+                  time_q << "(time > #{start - 1}s)"
+                end
+                unless finish.nil?
+                  time_q << "(time < #{finish}s)"
+                end
+              when :diff_range
+                unless start.nil?
+                  time_q << "(time < #{start}s)"
+                end
+                unless finish.nil?
+                  time_q << "(time > #{finish - 1}s)"
+                end
+              end
+
+              query += time_q.join(' AND ')
+            end
+
+            query += ")"
+            query
+          end
+        end
+
+        def self.evaluate(backend, op, associated_class, source, source_keys, temp_keys, opts = {})
+          shortcut = opts[:shortcut]
+
+          weights = case op
+          when :union, :union_range
+            [0.0] * source_keys.length
+          when :diff, :diff_range
+            [-1.0] * source_keys.length
+          end
+
+          r_source   = backend.key_to_redis_key(source)
+          r_source_keys = source_keys.collect {|sk| backend.key_to_redis_key(sk) }
+
+          dest_sorted_set = associated_class.send(:temp_key, :sorted_set)
+          temp_keys << dest_sorted_set
+          r_dest_sorted_set = backend.key_to_redis_key(dest_sorted_set)
+
+          case op
+          when :union, :union_range
+            Zermelo.redis.zinterstore(r_dest_sorted_set, r_source_keys, :weights => weights, :aggregate => 'max')
+            Zermelo.redis.zunionstore(r_dest_sorted_set, [r_source, r_dest_sorted_set])
+          when :intersect, :intersect_range
+            Zermelo.redis.zinterstore(r_dest_sorted_set, [r_source] + r_source_keys, :weights => weights, :aggregate => 'max')
+          when :diff, :diff_range
+            # 'zdiffstore' via weights, relies on non-zero scores being used
+            # see https://code.google.com/p/redis/issues/detail?id=579
+            Zermelo.redis.zinterstore(r_dest_sorted_set, r_source_keys, :weights => weights, :aggregate => 'max')
+            Zermelo.redis.zunionstore(r_dest_sorted_set, [r_source, r_dest_sorted_set])
+            Zermelo.redis.zremrangebyscore(r_dest_sorted_set, "0", "0")
+          end
+
+          return dest_sorted_set if shortcut.nil?
+          REDIS_SHORTCUTS[shortcut].call(*([r_dest_sorted_set] + opts[:shortcut_args]))
+        end
+
+      end
+    end
+  end
+end

data/lib/zermelo/records/class_methods.rb

@@ -102,13 +102,25 @@ module Zermelo
 
       private
 
+      def class_key
+        self.name.demodulize.underscore
+      end
+
       def ids_key
-        @ids_key ||= Zermelo::Records::Key.new(:klass => class_key, :name => 'ids',
-          :type => :set, :object => :attribute)
+        @ids_key ||= Zermelo::Records::Key.new(
+                       :klass => self, :name => 'ids',
+                       :type => :set,
+                       :object => :attribute
+                     )
       end
 
-      def class_key
-        self.name.demodulize.underscore
+      def temp_key(type)
+        Zermelo::Records::Key.new(
+          :klass  => self,
+          :name   => SecureRandom.hex(16),
+          :type   => type,
+          :object => :temporary
+        )
       end
 
       def load(id)

data/lib/zermelo/records/influxdb_record.rb

@@ -28,6 +28,8 @@ module Zermelo
 
       included do
         set_backend :influxdb
+
+        define_attributes :time => :timestamp
       end
 
     end

data/lib/zermelo/records/instance_methods.rb

@@ -51,7 +51,7 @@ module Zermelo
           attr_types = self.class.attribute_types.reject {|k, v| k == :id}
 
           attrs_to_load = attr_types.collect do |name, type|
-            Zermelo::Records::Key.new(:klass => class_key,
+            Zermelo::Records::Key.new(:klass => self.class,
               :id => self.id, :name => name, :type => type, :object => :attribute)
           end
 
@@ -167,12 +167,12 @@ module Zermelo
               end
 
               self.class.attribute_types.each_pair {|name, type|
-                key = Zermelo::Records::Key.new(:klass => self.class.send(:class_key),
+                key = Zermelo::Records::Key.new(:klass => self.class,
                   :id => self.id, :name => name.to_s, :type => type, :object => :attribute)
                 backend.clear(key)
               }
 
-              record_key = Zermelo::Records::Key.new(:klass => self.class.send(:class_key),
+              record_key = Zermelo::Records::Key.new(:klass => self.class,
                   :id => self.id)
               backend.purge(record_key)
             end
@@ -190,7 +190,7 @@ module Zermelo
         @attribute_keys ||= self.class.attribute_types.reject {|k, v|
           k == :id
         }.inject({}) {|memo, (name, type)|
-          memo[name.to_s] = Zermelo::Records::Key.new(:klass => self.class.send(:class_key),
+          memo[name.to_s] = Zermelo::Records::Key.new(:klass => self.class,
             :id => self.id, :name => name.to_s, :type => type, :object => :attribute)
           memo
         }

data/lib/zermelo/records/key.rb

@@ -1,5 +1,6 @@
 module Zermelo
   module Records
+
     class Key
 
       # id       / if nil, it's a class variable
@@ -15,5 +16,6 @@ module Zermelo
       end
 
     end
+
   end
 end

data/lib/zermelo/version.rb

@@ -1,3 +1,3 @@
 module Zermelo
-  VERSION = '1.0.1'
+  VERSION = '1.1.0'
 end

data/lib/zermelo.rb

@@ -95,7 +95,15 @@ module Zermelo
           debug_str
         }
       end
-      @proxied_connection.send(name, *args, &block)
+      result = @proxied_connection.send(name, *args, &block)
+      unless Zermelo.logger.nil?
+        Zermelo.logger.debug {
+          debug_str = "#{name}"
+          debug_str += " result: #{result}"
+          debug_str
+        }
+      end
+      result
     end
   end