zermelo 1.1.0 → 1.2.0

data/lib/zermelo/filters/{influxdb_filter.rb → influxdb.rb}

@@ -1,10 +1,10 @@
-require 'zermelo/filters/base'
+require 'zermelo/filter'
 
 module Zermelo
   module Filters
-    class InfluxDBFilter
+    class InfluxDB
 
-      include Zermelo::Filters::Base
+      include Zermelo::Filter
 
       private
 
@@ -35,6 +35,7 @@ module Zermelo
         end
       end
 
+      # TODO support the new before/after_read callbacks
       def resolve_steps(result_type)
         class_key = @associated_class.send(:class_key)
 
@@ -56,7 +57,7 @@ module Zermelo
           begin
             initial_id_data =
               Zermelo.influxdb.query(ii_query)["#{initial_class_key}/#{@initial_key.id}"]
-          rescue InfluxDB::Error => ide
+          rescue ::InfluxDB::Error => ide
             raise unless
               /^Field #{@initial_key.name} doesn't exist in series #{initial_class_key}\/#{@initial_key.id}$/ === ide.message
 
@@ -83,16 +84,19 @@ module Zermelo
 
           first_step = steps.first
 
+          attr_types = @associated_class.send(:attribute_types)
+
           query += @steps.collect {|step|
-            step.resolve(backend, @associated_class, :first => (step == first_step))
+            step.resolve(backend, @associated_class, :first => (step == first_step),
+              :attr_types  => attr_types)
           }.join("")
         end
 
-        query += " LIMIT 1"
+        query += " ORDER ASC LIMIT 1"
 
         begin
           result = Zermelo.influxdb.query(query)
-        rescue InfluxDB::Error => ide
+        rescue ::InfluxDB::Error => ide
           raise unless /^Couldn't look up columns$/ === ide.message
           result = {}
         end
@@ -103,10 +107,7 @@ module Zermelo
         when :ids
           data_keys.empty? ? [] : data_keys.collect {|k| k =~ /^#{class_key}\/(.+)$/; $1 }
         when :count
-          data_keys.empty? ?  0 : data_keys.inject(0) do |memo, k|
-            memo += result[k].first['count']
-            memo
-          end
+          data_keys.empty? ?  0 : data_keys.size
         end
       end
     end

data/lib/zermelo/filters/redis.rb

@@ -0,0 +1,173 @@
+require 'zermelo/filter'
+
+require 'zermelo/filters/index_range'
+
+# TODO check escaping of ids and index_keys -- shouldn't allow bare :
+
+module Zermelo
+
+  module Filters
+
+    class Redis
+
+      include Zermelo::Filter
+
+      SHORTCUTS = {
+        :list => {
+          :ids     => proc {|key|     Zermelo.redis.lrange(key, 0, -1) },
+          :count   => proc {|key|     Zermelo.redis.llen(key) },
+          :exists? => proc {|key, id| Zermelo.redis.lrange(key, 0, -1).include?(id) },
+          :first   => proc {|key|     Zermelo.redis.lrange(key, 0, 0).first },
+          :last    => proc {|key|     Zermelo.redis.lrevrange(key, 0, 0).first }
+        },
+        :set => {
+          :ids     => proc {|key|     Zermelo.redis.smembers(key) },
+          :count   => proc {|key|     Zermelo.redis.scard(key) },
+          :exists? => proc {|key, id| Zermelo.redis.sismember(key, id) }
+        },
+        :sorted_set => {
+          :ids     => proc {|key, order|
+            Zermelo.redis.send((:desc.eql?(order) ? :zrevrange : :zrange), key, 0, -1)
+          },
+          :count   => proc {|key, order|     Zermelo.redis.zcard(key) },
+          :exists? => proc {|key, order, id| !Zermelo.redis.zscore(key, id).nil? },
+          :first   => proc {|key, order|
+            Zermelo.redis.send((:desc.eql?(order) ? :zrevrange : :zrange), key, 0, 0).first
+          },
+          :last    => proc {|key, order|
+            Zermelo.redis.send((:desc.eql?(order) ? :zrange : :zrevrange), key, 0, 0).first
+          }
+        }
+      }
+
+      # TODO polite error when first/last applied to set
+
+      # more step users
+      def first
+        lock {
+          first_id = resolve_steps(:first)
+          first_id.nil? ? nil : _load(first_id)
+        }
+      end
+
+      def last
+        lock {
+          last_id = resolve_steps(:last)
+          last_id.nil? ? nil : _load(last_id)
+        }
+      end
+      # end step users
+
+      private
+
+      def _count
+        resolve_steps(:count)
+      end
+
+      def _ids
+        resolve_steps(:ids)
+      end
+
+      def _exists?(id)
+        return if id.nil?
+        resolve_steps(:exists?, id)
+      end
+
+      # If called with a block --  takes a block and passes the name of a set to
+      # it; deletes all temporary sets once done
+
+      # If called with any arguments -- treats them as a hash of shortcuts
+
+      # If not called with any arguments -- returns two values, the first is
+      # the name of a set containing the filtered ids, the second is a boolean
+      # for whether or not to clear up that set once it's been used
+
+      def resolve_steps(shortcut, *args)
+        if @steps.empty?
+          unless @callback_target_class.nil? || @callbacks.nil?
+            br = @callbacks[:before_read]
+            if !br.nil? && @callback_target_class.respond_to?(br)
+              @callback_target_class.send(br, @callback_target_id)
+            end
+          end
+
+          sc = Zermelo::Filters::Redis::SHORTCUTS[@initial_key.type][shortcut]
+          ret = if sc.nil?
+            yield(@initial_key)
+          else
+            r_key = backend.key_to_redis_key(@initial_key)
+            shortcut_params = if @initial_key.type == :sorted_set
+              [r_key, @sort_order] + args
+            else
+              [r_key] + args
+            end
+            sc.call(*shortcut_params)
+          end
+
+          unless @callback_target_class.nil? || @callbacks.nil?
+            ar = @callbacks[:after_read]
+            if !ar.nil? && @callback_target_class.respond_to?(ar)
+              @callback_target_class.send(ar, @callback_target_id)
+            end
+          end
+
+          return(ret)
+        end
+
+        idx_attrs = @associated_class.send(:with_index_data) do |d|
+          d.each_with_object({}) do |(name, data), memo|
+            memo[name.to_s] = data.index_klass
+          end
+        end
+
+        attr_types = @associated_class.send(:attribute_types)
+
+        backend.temp_key_wrap do |temp_keys|
+          result     = nil
+          last_step  = @steps.last
+
+          step_opts = {
+            :index_attrs => idx_attrs,
+            :attr_types  => attr_types,
+            :temp_keys   => temp_keys,
+            :source      => @initial_key,
+            :initial_key => @initial_key,
+            :sort_order  => @sort_order
+          }
+
+          @steps.each do |step|
+            unless step.class.accepted_types.include?(step_opts[:source].type)
+              raise "'#{step.class.name}' does not accept input type #{step_opts[:source].type}"
+            end
+
+            if step == last_step
+              step_opts.update(:shortcut => shortcut, :shortcut_args => args)
+            end
+
+            unless @callback_target_class.nil? || @callbacks.nil?
+              br = @callbacks[:before_read]
+              if !br.nil? && @callback_target_class.respond_to?(br)
+                @callback_target_class.send(br, @callback_target_id)
+              end
+            end
+
+            result = step.resolve(backend, @associated_class, step_opts)
+
+            unless @callback_target_class.nil? || @callbacks.nil?
+              ar = @callbacks[:after_read]
+              if !ar.nil? && @callback_target_class.respond_to?(ar)
+                @callback_target_class.send(ar, @callback_target_id)
+              end
+            end
+
+            step_opts[:source] = result unless step == last_step
+          end
+
+          result
+        end
+      end
+    end
+
+  end
+
+end

data/lib/zermelo/filters/steps/list_step.rb

@@ -5,21 +5,13 @@ module Zermelo
     class Steps
       class ListStep < Zermelo::Filters::Steps::BaseStep
         def self.accepted_types
-          [:list]
+          [:sorted_set, :list]
         end
 
         def self.returns_type
-          :list
+          nil # same as the source type
         end
 
-        REDIS_SHORTCUTS = {
-          :ids     => proc {|key|     Zermelo.redis.lrange(key, 0, -1) },
-          :count   => proc {|key|     Zermelo.redis.llen(key) },
-          :exists? => proc {|key, id| Zermelo.redis.lrange(key, 0, -1).include?(id) },
-          :first   => proc {|key|     Zermelo.redis.lrange(key, 0, 0).first },
-          :last    => proc {|key|     Zermelo.redis.lrevrange(key, 0, 0).first }
-        }
-
         def resolve(backend, associated_class, opts = {})
           shortcut = opts[:shortcut]
 
@@ -30,46 +22,72 @@ module Zermelo
           l = limit.to_i
 
           case backend
-          when Zermelo::Backends::RedisBackend
+          when Zermelo::Backends::Redis
 
             source = opts[:source]
             idx_attrs = opts[:index_attrs]
             attr_types = opts[:attr_types]
             temp_keys = opts[:temp_keys]
 
-          # TODO apply these transformations via a subset?
-          # TODO need a guaranteed non-existing key for non-sorting 'sort'
+            # TODO apply these transformations via a subset?
+            # TODO need a guaranteed non-existing key for non-sorting 'sort'
 
             # TODO check if source is in temp_keys, use a generated temp_key instead if not
             r_source = backend.key_to_redis_key(source)
 
-            l = (Zermelo.redis.llen(r_source) - o) if (l < 1)
+            result, r_result = case source.type
+            when :sorted_set
+              limited = associated_class.send(:temp_key, :sorted_set)
+              temp_keys << limited
+              r_limited = backend.key_to_redis_key(limited)
+
+              lim = Zermelo.redis.send(
+                :desc.eql?(opts[:sort_order]) ? :zrevrange : :zrange,
+                r_source, o, (o + l), :with_scores => true
+              )
+
+              Zermelo.redis.zadd(r_limited, lim.collect {|l| [l[1], l[0]]} )
+
+              [limited, r_limited]
+            when :list
 
-            sort_opts = {:by => 'no_sort', :limit => [o, l]}
+              l = (Zermelo.redis.llen(r_source) - o) if (l < 1)
 
-            # https://github.com/antirez/redis/issues/2079, fixed in redis 2.8.19
-            result, r_result = if (Zermelo.redis_version.split('.') <=> ['2', '8', '18']) == 1
-              sort_opts.update(:store => r_source)
-              Zermelo.redis.sort(r_source, sort_opts)
-              [source, r_source]
-            else
-              data = Zermelo.redis.sort(r_source, sort_opts)
+              sort_opts = {:by => 'no_sort', :limit => [o, l]}
 
-              if data.empty?
-                # TODO fix
+              # https://github.com/antirez/redis/issues/2079, fixed in redis 2.8.19
+              result, r_result = if (Zermelo.redis_version.split('.') <=> ['2', '8', '18']) == 1
+                sort_opts.update(:store => r_source)
+                Zermelo.redis.sort(r_source, sort_opts)
+                [source, r_source]
               else
-                limited = associated_class.send(:temp_key, :list)
-                temp_keys << limited
-                r_limited = backend.key_to_redis_key(limited)
+                data = Zermelo.redis.sort(r_source, sort_opts)
 
-                Zermelo.redis.rpush(r_limited, data)
+                if data.empty?
+                  # TODO fix
+                else
+                  limited = associated_class.send(:temp_key, :list)
+                  temp_keys << limited
+                  r_limited = backend.key_to_redis_key(limited)
 
-                [limited, r_limited]
+                  Zermelo.redis.rpush(r_limited, data)
+
+                  [limited, r_limited]
+                end
               end
             end
 
             return result if shortcut.nil?
-            REDIS_SHORTCUTS[shortcut].call(*([r_result] + opts[:shortcut_args]))
+
+            extra_args = case source.type
+            when :sorted_set
+              [opts[:sort_order]]
+            when :list
+              []
+            end
+
+            Zermelo::Filters::Redis::SHORTCUTS[source.type][shortcut].
+              call(*([r_result] + extra_args + opts[:shortcut_args]))
           end
         end
       end

data/lib/zermelo/filters/steps/set_step.rb

@@ -6,29 +6,25 @@ module Zermelo
     class Steps
       class SetStep < Zermelo::Filters::Steps::BaseStep
         def self.accepted_types
-          [:set, :sorted_set] # TODO should allow :list as well
+          [:set, :sorted_set] # TODO should allow :list as well?
         end
 
         def self.returns_type
-          :set
+          nil # same as the source type
         end
 
-        REDIS_SHORTCUTS = {
-          :ids     => proc {|key|     Zermelo.redis.smembers(key) },
-          :count   => proc {|key|     Zermelo.redis.scard(key) },
-          :exists? => proc {|key, id| Zermelo.redis.sismember(key, id) }
-        }
-
         def resolve(backend, associated_class, opts = {})
 
           case backend
-          when Zermelo::Backends::RedisBackend
-            source     = opts[:source]
-            idx_attrs  = opts[:index_attrs]
-            attr_types = opts[:attr_types]
-            temp_keys  = opts[:temp_keys]
+          when Zermelo::Backends::Redis
+            initial_key = opts[:initial_key]
+            source      = opts[:source]
+            idx_attrs   = opts[:index_attrs]
+            attr_types  = opts[:attr_types]
+            temp_keys   = opts[:temp_keys]
+            order       = opts[:sort_order]
 
-            source_keys = @attributes.inject([]) do |memo, (att, value)|
+            source_keys = @attributes.each_with_object([]) do |(att, value), memo|
 
               val = value.is_a?(Set) ? value.to_a : value
 
@@ -42,7 +38,7 @@ module Zermelo
                 raise "'#{att}' property is not indexed" if idx_class.nil?
 
                 if val.is_a?(Enumerable)
-                  conditions_set = associated_class.send(:temp_key, :set)
+                  conditions_set = associated_class.send(:temp_key, source.type)
                   r_conditions_set = backend.key_to_redis_key(conditions_set)
 
                   backend.temp_key_wrap do |conditions_temp_keys|
@@ -52,7 +48,12 @@ module Zermelo
                       backend.key_to_redis_key(il)
                     }
 
-                    Zermelo.redis.sunionstore(r_conditions_set, *index_keys)
+                    case source.type
+                    when :set
+                      Zermelo.redis.sunionstore(r_conditions_set, *index_keys)
+                    when :sorted_set
+                      Zermelo.redis.zunionstore(r_conditions_set, index_keys)
+                    end
                   end
                   memo << conditions_set
                 else
@@ -60,22 +61,90 @@ module Zermelo
                             idx_class, val, attr_types[att], temp_keys)
                 end
               end
-
-              memo
             end
 
-            case source.type
-            when :sorted_set
-              Zermelo::Filters::Steps::SortedSetStep.evaluate(backend,
-                @options[:op], associated_class, source, source_keys, temp_keys, opts)
-            when :set
-              self.class.evaluate(backend, @options[:op], associated_class,
-                source, source_keys, temp_keys, opts)
+            r_source_key  = backend.key_to_redis_key(source)
+            r_source_keys = source_keys.collect {|sk| backend.key_to_redis_key(sk) }
+
+            op = @options[:op]
+            shortcut = opts[:shortcut]
+
+            if :ids.eql?(shortcut) && (source.type == :set)
+              case op
+              when :union
+                backend.temp_key_wrap do |shortcut_temp_keys|
+                  dest_set = associated_class.send(:temp_key, :set)
+                  shortcut_temp_keys << dest_set
+                  r_dest_set = backend.key_to_redis_key(dest_set)
+
+                  Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+                  Zermelo.redis.sunion(r_source_key, r_dest_set)
+                end
+              when :intersect
+                Zermelo.redis.sinter(r_source_key, *r_source_keys)
+              when :diff
+                backend.temp_key_wrap do |shortcut_temp_keys|
+                  dest_set = associated_class.send(:temp_key, :set)
+                  shortcut_temp_keys << dest_set
+                  r_dest_set = backend.key_to_redis_key(dest_set)
+
+                  Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+                  Zermelo.redis.sdiff(r_source_key, r_dest_set)
+                end
+              end
+            else
+              dest_set = associated_class.send(:temp_key, source.type)
+              r_dest_set = backend.key_to_redis_key(dest_set)
+              temp_keys << dest_set
+
+              case op
+              when :union
+                r_initial_key = backend.key_to_redis_key(initial_key)
+
+                if source.type == :sorted_set
+                  Zermelo.redis.zinterstore(r_dest_set,
+                    [r_initial_key] + r_source_keys,
+                    :weights => [1.0] + ([0.0] * source_keys.length), :aggregate => 'max')
+
+                  Zermelo.redis.zunionstore(r_dest_set, [r_source_key, r_dest_set], :aggregate => 'max')
+                else
+                  Zermelo.redis.sinterstore(r_dest_set, r_initial_key, *r_source_keys)
+                  Zermelo.redis.sunionstore(r_dest_set, r_dest_set, r_source_key)
+                end
+              when :intersect
+                if source.type == :sorted_set
+                  Zermelo.redis.zinterstore(r_dest_set, [r_source_key] + r_source_keys, :aggregate => 'max')
+                else
+                  Zermelo.redis.sinterstore(r_dest_set, r_source_key, *r_source_keys)
+                end
+              when :diff
+                if source.type == :sorted_set
+                  Zermelo.redis.zinterstore(r_dest_set, r_source_keys, :aggregate => 'max')
+                  Zermelo.redis.zunionstore(r_dest_set, [r_source_key, r_dest_set], :weights => [1.0, 0.0], :aggregate => 'min')
+                  Zermelo.redis.zremrangebyscore(r_dest_set, "0", "0")
+                else
+                  Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
+                  Zermelo.redis.sdiffstore(r_dest_set, r_dest_set, r_source_key)
+                end
+              end
+
+              return dest_set if shortcut.nil?
+
+              shortcut_params = if source.type == :sorted_set
+                [r_dest_set, order] + opts[:shortcut_args]
+              else
+                [r_dest_set] + opts[:shortcut_args]
+              end
+
+              Zermelo::Filters::Redis::SHORTCUTS[source.type][shortcut].
+                call(*shortcut_params)
             end
 
-          when Zermelo::Backends::InfluxDBBackend
+          when Zermelo::Backends::InfluxDB
             query = ''
 
+            attr_types = opts[:attr_types]
+
             unless opts[:first].is_a?(TrueClass)
               case @options[:op]
               when :intersect, :diff
@@ -88,27 +157,68 @@ module Zermelo
             case @options[:op]
             when :intersect, :union
               query += @attributes.collect {|k, v|
-                op, value = case v
-                when String
-                  ["=~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+
+                attr_type = attr_types[k]
+
+                if v.is_a?(Enumerable)
+                  qq = v.each_with_object([]) do |vv, memo|
+                    ov = case vv
+                    when Regexp
+                      raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
+                      "=~ /#{vv.source.gsub(/\\\\/, "\\")}/"
+                    when String
+                      "=~ /^#{Regexp.escape(vv).gsub(/\\\\/, "\\")}$/"
+                    else
+                      "= '#{vv}'"
+                    end
+                    memo << "#{k} #{ov}"
+                  end
+                  "((#{qq.join(') OR (')}))"
                 else
-                  ["=",  "'#{v}'"]
+                  op_value = case v
+                  when Regexp
+                    raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
+                    "=~ /#{v.source.gsub(/\\\\/, "\\")}/"
+                  when String
+                    "=~ /^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"
+                  else
+                    "= '#{v}'"
+                  end
+                  "(#{k} #{op_value})"
                 end
-
-               "#{k} #{op} #{value}"
               }.join(' AND ')
 
             when :diff
               query += @attributes.collect {|k, v|
-                op, value = case v
-                when String
-                  ["!~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+                if v.is_a?(Enumerable)
+                  qq = v.each_with_object([]) do |vv, memo|
+                    ov = case vv
+                    when Regexp
+                      raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
+                      "!~ /#{vv.source.gsub(/\\\\/, "\\")}/"
+                    when String
+                      "!~ /^#{Regexp.escape(vv).gsub(/\\\\/, "\\")}$/"
+                    else
+                      "<> '#{vv}'"
+                    end
+                    memo << "#{k} #{ov}"
+                  end
+                  "((#{qq.join(') OR (')}))"
                 else
-                  ["!=",  "'#{v}'"]
-                end
+                  op_value = case v
+                  when Regexp
+                    raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
+                    "!~ /#{v.source.gsub(/\\\\/, "\\")}/"
+                  when String
+                    "!~ /^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"
+                  else
+                    "<> '#{v}'"
+                  end
 
-                "#{k} #{op} #{value}"
+                  "(#{k} #{op_value})"
+                end
               }.join(' AND ')
+
             else
               raise "Unhandled filter operation '#{@options[:op]}'"
             end
@@ -119,57 +229,6 @@ module Zermelo
           end
         end
 
-        def self.evaluate(backend, op, associated_class, source, source_keys, temp_keys, opts = {})
-          shortcut = opts[:shortcut]
-
-          r_source_key  = backend.key_to_redis_key(source)
-          r_source_keys = source_keys.collect {|sk| backend.key_to_redis_key(sk) }
-
-          if :ids.eql?(shortcut)
-            case op
-            when :union
-              backend.temp_key_wrap do |shortcut_temp_keys|
-                dest_set = associated_class.send(:temp_key, :set)
-                shortcut_temp_keys << dest_set
-                r_dest_set = backend.key_to_redis_key(dest_set)
-
-                Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
-                Zermelo.redis.sunion(r_dest_set, r_source_key)
-              end
-            when :intersect
-              Zermelo.redis.sinter(r_source_key, *r_source_keys)
-            when :diff
-              backend.temp_key_wrap do |shortcut_temp_keys|
-                dest_set = associated_class.send(:temp_key, :set)
-                shortcut_temp_keys << dest_set
-                r_dest_set = backend.key_to_redis_key(dest_set)
-
-                Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
-                Zermelo.redis.sdiff(r_source_key, r_dest_set)
-              end
-            end
-          else
-            dest_set = associated_class.send(:temp_key, :set)
-            r_dest_set = backend.key_to_redis_key(dest_set)
-            temp_keys << dest_set
-
-            case op
-            when :union
-              Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
-              Zermelo.redis.sunionstore(r_dest_set, r_source_key, r_dest_set)
-            when :intersect
-              Zermelo.redis.sinterstore(r_dest_set, r_source_key, *r_source_keys)
-            when :diff
-              Zermelo.redis.sinterstore(r_dest_set, *r_source_keys)
-              Zermelo.redis.sdiffstore(r_dest_set, r_source_key, r_dest_set)
-            end
-
-            return dest_set if shortcut.nil?
-            REDIS_SHORTCUTS[shortcut].call(*([r_dest_set] + opts[:shortcut_args]))
-          end
-
-        end
-
       end
     end
   end

data/lib/zermelo/filters/steps/sort_step.rb

@@ -14,7 +14,7 @@ module Zermelo
 
         def resolve(backend, associated_class, opts = {})
           case backend
-          when Zermelo::Backends::RedisBackend
+          when Zermelo::Backends::Redis
             source = opts[:source]
             idx_attrs = opts[:index_attrs]
             attr_types = opts[:attr_types]
@@ -92,7 +92,7 @@ module Zermelo
             shortcut = opts[:shortcut]
 
             return dest_list if shortcut.nil?
-            Zermelo::Filters::Steps::ListStep::REDIS_SHORTCUTS[shortcut].
+            Zermelo::Filters::Redis::SHORTCUTS[:list][shortcut].
               call(*([r_dest_list] + opts[:shortcut_args]))
           end
         end