zermelo 1.1.0 → 1.2.0

data/lib/zermelo/backends/{influxdb_backend.rb → influxdb.rb}

@@ -1,6 +1,6 @@
-require 'zermelo/backends/base'
+require 'zermelo/backend'
 
-require 'zermelo/filters/influxdb_filter'
+require 'zermelo/filters/influxdb'
 
 # NB influxdb doesn't support individually addressable deletes, so
 # this backend only works to write new records
@@ -11,16 +11,19 @@ module Zermelo
 
   module Backends
 
-    class InfluxDBBackend
+    class InfluxDB
 
-      include Zermelo::Backends::Base
+      include Zermelo::Backend
 
-      def default_sorted_set_key
-        :time
-      end
+      # def default_sorted_set_key
+      #   :time
+      # end
+
+      def filter(ids_key, associated_class, callback_target_class = nil,
+                 callback_target_id = nil, callbacks = nil, sort_order = nil)
 
-      def filter(ids_key, record)
-        Zermelo::Filters::InfluxDBFilter.new(self, ids_key, record)
+        Zermelo::Filters::InfluxDB.new(self, ids_key, associated_class,
+          callback_target_class, callback_target_id, callbacks, sort_order)
       end
 
       # TODO get filter calling this instead of using same logic
@@ -38,7 +41,7 @@ module Zermelo
           begin
           records = Zermelo.influxdb.query("SELECT #{attr_key.name} FROM " +
             "\"#{class_key}/#{attr_key.id}\" LIMIT 1")["#{class_key}/#{attr_key.id}"]
-          rescue InfluxDB::Error => ide
+          rescue ::InfluxDB::Error => ide
             raise unless
               /^Field #{attr_key.name} doesn't exist in series #{class_key}\/#{attr_key.id}$/ === ide.message
 
@@ -50,9 +53,16 @@ module Zermelo
           memo[class_key][attr_key.id] ||= {}
 
           memo[class_key][attr_key.id][attr_key.name.to_s] = if value.nil?
-            nil
+            case attr_key.type
+            when :list
+              []
+            when :hash
+              {}
+            when :set
+              Set.new()
+            else nil
+            end
           else
-
             case attr_key.type
             when :string
               value.to_s
@@ -130,11 +140,29 @@ module Zermelo
           class_key = key.klass.send(:class_key)
 
           records[class_key]         ||= {}
-          records[class_key][key.id] ||= {}
 
-          records[class_key][key.id][key.name] = case op
+          if records[class_key][key.id].nil?
+            begin
+              result = Zermelo.influxdb.query("SELECT * FROM \"#{class_key}/#{key.id}\" LIMIT 1")["#{class_key}/#{key.id}"]
+              if result.nil?
+                records[class_key][key.id] = {}
+              else
+                records[class_key][key.id] = result.first
+                records[class_key][key.id].delete_if {|k,v| ["time", "sequence_number"].include?(k) }
+              end
+            rescue ::InfluxDB::Error => ide
+              raise unless
+                (/^Couldn't look up columns for series: #{class_key}\/#{key.id}$/ === ide.message) ||
+                (/^Couldn't look up columns$/ === ide.message) ||
+                (/^Couldn't find series: #{class_key}\/#{key.id}$/ === ide.message)
+
+              records[class_key][key.id] = {}
+            end
+          end
+
+          case op
           when :set
-            case key.type
+            records[class_key][key.id][key.name] = case key.type
             when :string, :integer
               value.nil? ? nil : value.to_s
             when :timestamp
@@ -148,12 +176,51 @@ module Zermelo
             end
           when :add
             case key.type
-            when :list, :hash
-              value
+            when :hash
+              if records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] = value
+              else
+                records[class_key][key.id][key.name].update(value)
+              end
+            when :list
+              if records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] = value
+              else
+                records[class_key][key.id][key.name] += value
+              end
             when :set
-              value.to_a
+              v = value.to_a
+              if records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] = v
+              else
+                records[class_key][key.id][key.name] += v
+              end
+            when :sorted_set
+              v = (1...value.size).step(2).collect {|i| value[i] }
+              if records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] = v
+              else
+                records[class_key][key.id][key.name] += v
+              end
+            end
+          when :delete
+            case key.type
+            when :hash
+              unless records[class_key][key.id][key.name].nil?
+                # FIXME
+              end
+            when :list
+              unless records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] -= value
+              end
+            when :set
+              unless records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] -= value.to_a
+              end
             when :sorted_set
-              (1...value.size).step(2).collect {|i| value[i] }
+              unless records[class_key][key.id][key.name].nil?
+                records[class_key][key.id][key.name] -= (1...value.size).step(2).collect {|i| value[i] }
+              end
             end
           when :purge
             purges << "\"#{class_key}/#{key.id}\""
@@ -163,19 +230,8 @@ module Zermelo
 
         records.each_pair do |class_key, klass_records|
           klass_records.each_pair do |id, data|
-            begin
-              prior = Zermelo.influxdb.query("SELECT * FROM \"#{class_key}/#{id}\" LIMIT 1")["#{class_key}/#{id}"]
-            rescue InfluxDB::Error => ide
-              raise unless
-                (/^Couldn't look up columns for series: #{class_key}\/#{id}$/ === ide.message) ||
-                (/^Couldn't look up columns$/ === ide.message) ||
-                (/^Couldn't find series: #{class_key}\/#{id}$/ === ide.message)
-
-              prior = nil
-            end
-            record = prior.nil? ? {} : prior.first.delete_if {|k,v| ["time", "sequence_number"].include?(k) }
             data.delete('time') if data.has_key?('time') && data['time'].nil?
-            Zermelo.influxdb.write_point("#{class_key}/#{id}", record.merge(data).merge('id' => id))
+            Zermelo.influxdb.write_point("#{class_key}/#{id}", data.merge('id' => id))
           end
         end
 

data/lib/zermelo/backends/{redis_backend.rb → redis.rb}

@@ -1,26 +1,30 @@
-require 'zermelo/backends/base'
+require 'zermelo/backend'
 
-require 'zermelo/filters/redis_filter'
+require 'zermelo/filters/redis'
 require 'zermelo/locks/redis_lock'
 
 module Zermelo
 
   module Backends
 
-    class RedisBackend
+    class Redis
 
-      include Zermelo::Backends::Base
+      include Zermelo::Backend
 
-      def default_sorted_set_key
-        :timestamp
-      end
+      # def default_sorted_set_key
+      #   :timestamp
+      # end
 
       def generate_lock
         Zermelo::Locks::RedisLock.new
       end
 
-      def filter(ids_key, record)
-        Zermelo::Filters::RedisFilter.new(self, ids_key, record)
+      def filter(ids_key, associated_class, callback_target_class = nil,
+        callback_target_id = nil, callbacks = nil, sort_order = nil)
+
+        Zermelo::Filters::Redis.new(self, ids_key, associated_class,
+                                    callback_target_class, callback_target_id,
+                                    callbacks, sort_order)
       end
 
       def get_multiple(*attr_keys)
@@ -38,26 +42,25 @@ module Zermelo
               if attr_key.accessor.nil?
                 Zermelo.redis.lrange(redis_attr_key, 0, -1)
               else
-
+                # TODO
               end
             when :set
               if attr_key.accessor.nil?
                 Set.new( Zermelo.redis.smembers(redis_attr_key) )
               else
-
+                # TODO
               end
             when :hash
               if attr_key.accessor.nil?
                 Zermelo.redis.hgetall(redis_attr_key)
               else
-
+                # TODO
               end
             when :sorted_set
-              # TODO should this be something that preserves order?
               if attr_key.accessor.nil?
-                Set.new( Zermelo.redis.zrange(redis_attr_key, 0, -1) )
+                Zermelo.redis.zrange(redis_attr_key, 0, -1)
               else
-
+                # TODO
               end
             end
 
@@ -94,19 +97,6 @@ module Zermelo
         end
       end
 
-      def exists?(key)
-        Zermelo.redis.exists(key_to_redis_key(key))
-      end
-
-      def include?(key, id)
-        case key.type
-        when :set
-          Zermelo.redis.sismember(key_to_redis_key(key), id)
-        else
-          raise "Not implemented"
-        end
-      end
-
       def begin_transaction
         return false unless @transaction_redis.nil?
         @transaction_redis = Zermelo.redis
@@ -169,22 +159,11 @@ module Zermelo
       # TODO converge usage of idx_class and _index lookup invocation
 
       def index_lookup(att, associated_class, idx_class, value, attr_type, temp_keys)
-        case value
-        # when Zermelo::Filters::RedisFilter
-
-          # TODO (maybe) if a filter from different backend, resolve to ids and
-          # put that in a Redis temp set
-
-          # collection, should_be_deleted = value.resolve_steps
-
-          # if should_be_deleted
-          #   temp_sets << collection.name
-          # end
-
-          # unless :set.eql?(collection.type)
-          #   raise "Unsure as yet if non-sets are safe as Filter step values"
-          # end
+        if (idx_class == Zermelo::Associations::RangeIndex) && !value.is_a?(Zermelo::Filters::IndexRange)
+          raise "Range index must be passed a range"
+        end
 
+        case value
         when Regexp
           raise "Can't query non-string values via regexp" unless :string.eql?(attr_type)
 
@@ -228,13 +207,31 @@ module Zermelo
           index = associated_class.send("#{att}_index")
 
           case index
+          when Zermelo::Associations::RangeIndex
+            r_index_key = key_to_redis_key(index.key)
+            range = if value.by_score
+              range_start  = value.start.nil?  ? '-inf' : safe_value(attr_type, value.start)
+              range_finish = value.finish.nil? ? '+inf' : safe_value(attr_type, value.finish)
+              Zermelo.redis.zrangebyscore(r_index_key, range_start, range_finish)
+            else
+              range_start  = value.start  ||  0
+              range_finish = value.finish || -1
+              Zermelo.redis.zrange(r_index_key, range_start, range_finish)
+            end
+
+            # TODO another way for index_lookup to indicate 'empty result', rather
+            # than creating & returning an empty key
+            idx_key = associated_class.send(:temp_key, :set)
+            temp_keys << idx_key
+            Zermelo.redis.sadd(key_to_redis_key(idx_key), range) unless range.empty?
+            idx_key
           when Zermelo::Associations::UniqueIndex
             idx_key = associated_class.send(:temp_key, :set)
             temp_keys << idx_key
 
             Zermelo.redis.sadd(key_to_redis_key(idx_key),
               Zermelo.redis.hget(key_to_redis_key(index.key),
-                                   index_keys(attr_type, value).join(':')))
+                                 index_keys(attr_type, value).join(':')))
             idx_key
           when Zermelo::Associations::Index
             index.key(value)
@@ -244,8 +241,8 @@ module Zermelo
 
       private
 
-      def change(op, key, value = nil, key_to = nil)
-        ch = [op, key, value, key_to]
+      def change(op, key, value = nil, key_to = nil, value_to = nil)
+        ch = [op, key, value, key_to, value_to]
         if @in_transaction
           @changes << ch
         else
@@ -259,10 +256,11 @@ module Zermelo
         purges = []
 
         changes.each do |ch|
-          op     = ch[0]
-          key    = ch[1]
-          value  = ch[2]
-          key_to = ch[3]
+          op       = ch[0]
+          key      = ch[1]
+          value    = ch[2]
+          key_to   = ch[3]
+          value_to = ch[4]
 
           # TODO check that collection types handle nil value for whole thing
           if Zermelo::COLLECTION_TYPES.has_key?(key.type)
@@ -302,16 +300,14 @@ module Zermelo
               when :set
                 Zermelo.redis.smove(complex_attr_key, key_to_redis_key(key_to), value)
               when :list
-                # TODO would do via sort 'nosort', except for
-                # https://github.com/antirez/redis/issues/2079 -- instead,
-                # copy the workaround from redis_filter.rb
+                # TODO via sort 'nosort', or the workaround required prior to
+                # https://github.com/antirez/redis/issues/2079
                 raise "Not yet implemented"
               when :hash
-                values = value.to_a.flatten
-                Zermelo.redis.hdel(complex_attr_key, values)
-                Zermelo.redis.hset(key_to_redis_key(key_to), *values)
+                Zermelo.redis.hdel(complex_attr_key, *value.keys)
+                Zermelo.redis.hset(key_to_redis_key(key_to), *value_to.to_a.flatten)
               when :sorted_set
-                raise "Not yet implemented"
+                Zermelo.redis.zadd(complex_attr_key, *value_to)
               end
             when :delete
               case key.type
@@ -329,7 +325,6 @@ module Zermelo
             end
 
           elsif :purge.eql?(op)
-            # TODO get keys for all assocs & indices, purge them too
             purges << ["#{key.klass.send(:class_key)}:#{key.id}:attrs"]
           else
             simple_attr_key = key_to_redis_key(key)

data/lib/zermelo/backends/stub.rb

@@ -0,0 +1,43 @@
+require 'zermelo/backend'
+
+module Zermelo
+
+  module Backends
+
+    class Stub
+
+      include Zermelo::Backend
+
+      def filter(ids_key, associated_class, callback_target_class = nil,
+        callback_target_id = nil, callbacks = nil, sort_order = nil)
+
+        raise "Not supported"
+      end
+
+      def get_multiple(*attr_keys)
+        raise "Not supported"
+      end
+
+      def begin_transaction
+        raise "Not supported"
+      end
+
+      def commit_transaction
+        raise "Not supported"
+      end
+
+      def abort_transaction
+        raise "Not supported"
+      end
+
+      private
+
+      def change(op, key, value = nil, key_to = nil, value_to = nil)
+        # no-op
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/filter.rb

@@ -0,0 +1,194 @@
+require 'active_support/concern'
+
+require 'zermelo/records/errors'
+
+require 'zermelo/filters/steps/list_step'
+require 'zermelo/filters/steps/set_step'
+require 'zermelo/filters/steps/sort_step'
+
+module Zermelo
+
+  module Filter
+
+    extend ActiveSupport::Concern
+
+    attr_reader :backend, :steps
+
+    # initial set         a Zermelo::Record::Key object
+    # associated_class    the class of the result record
+    # TODO hash for these params as it's getting too long
+    def initialize(data_backend, initial_key, associated_class,
+                   callback_target_class = nil, callback_target_id = nil,
+                   callbacks = nil, sort_order = nil,
+                   ancestor = nil, step = nil)
+      @backend          = data_backend
+      @initial_key      = initial_key
+      @associated_class = associated_class
+      @callback_target_class  = callback_target_class
+      @callback_target_id = callback_target_id
+      @callbacks        = callbacks
+      @sort_order       = sort_order
+      @steps            = ancestor.nil? ? [] : ancestor.steps.dup
+      @steps << step unless step.nil?
+    end
+
+    def intersect(attrs = {})
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::SetStep.new({:op => :intersect}, attrs))
+    end
+
+    def union(attrs = {})
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::SetStep.new({:op => :union}, attrs))
+    end
+
+    def diff(attrs = {})
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::SetStep.new({:op => :diff}, attrs))
+    end
+
+    def sort(keys, opts = {})
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::SortStep.new({:keys => keys,
+          :desc => opts[:desc], :limit => opts[:limit],
+          :offset => opts[:offset]}, {})
+        )
+    end
+
+    def offset(amount, opts = {})
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::ListStep.new({:offset => amount,
+          :limit => opts[:limit]}, {}))
+    end
+
+    # (a different syntax to the above)
+    def page(num, opts = {})
+      per_page = opts[:per_page].to_i || 20
+      start  = per_page * (num - 1)
+      self.class.new(@backend, @initial_key, @associated_class,
+        @callback_target_class, @callback_target_id, @callbacks, @sort_order,
+        self, ::Zermelo::Filters::Steps::ListStep.new({:offset => start,
+          :limit => per_page}, {}))
+    end
+
+    # step users
+    def exists?(e_id)
+      lock(false) { _exists?(e_id) }
+    end
+
+    def find_by_id(f_id)
+      lock { _find_by_id(f_id) }
+    end
+
+    def find_by_id!(f_id)
+      ret = find_by_id(f_id)
+      raise ::Zermelo::Records::Errors::RecordNotFound.new(@associated_class, f_id) if ret.nil?
+      ret
+    end
+
+    def find_by_ids(*f_ids)
+      lock { f_ids.collect {|f_id| _find_by_id(f_id) } }
+    end
+
+    def find_by_ids!(*f_ids)
+      ret = find_by_ids(*f_ids)
+      if ret.any? {|r| r.nil? }
+        raise ::Zermelo::Records::Errors::RecordsNotFound.new(@associated_class, f_ids - ret.compact.map(&:id))
+      end
+      ret
+    end
+
+    def ids
+      lock(false) { _ids }
+    end
+
+    def count
+      lock(false) { _count }
+    end
+
+    def empty?
+      lock(false) { _count == 0 }
+    end
+
+    def all
+      lock { _all }
+    end
+
+    def collect(&block)
+      lock { _ids.collect {|id| block.call(_load(id))} }
+    end
+    alias_method :map, :collect
+
+    def each(&block)
+      lock { _ids.each {|id| block.call(_load(id)) } }
+    end
+
+    def select(&block)
+      lock { _all.select {|obj| block.call(obj) } }
+    end
+    alias_method :find_all, :select
+
+    def reject(&block)
+      lock { _all.reject {|obj| block.call(obj)} }
+    end
+
+    def destroy_all
+      lock(*@associated_class.send(:associated_classes)) do
+        _all.each {|r| r.destroy }
+      end
+    end
+
+    def associated_ids_for(name, options = {})
+      data_type, type_klass = @associated_class.send(:with_association_data, name.to_sym) do |data|
+        [data.data_type, data.type_klass]
+      end
+
+      lock {
+        case data_type
+        when :belongs_to, :has_one
+          type_klass.send(:associated_ids_for, @backend, data_type,
+            @associated_class, name,
+            options[:inversed].is_a?(TrueClass), *_ids)
+        else
+          type_klass.send(:associated_ids_for, @backend, data_type,
+            @associated_class, name, *_ids)
+        end
+      }
+    end
+
+    protected
+
+    def lock(when_steps_empty = true, *klasses, &block)
+      return(block.call) if !when_steps_empty && @steps.empty?
+      klasses += [@associated_class] if !klasses.include?(@associated_class)
+      @backend.lock(*klasses, &block)
+    end
+
+    private
+
+    def _find_by_id(id)
+      if !id.nil? && _exists?(id)
+        _load(id.to_s)
+      else
+        nil
+      end
+    end
+
+    def _load(id)
+      object = @associated_class.new
+      object.load(id)
+      object
+    end
+
+    def _all
+      _ids.map {|id| _load(id) }
+    end
+
+  end
+
+end

data/lib/zermelo/filters/index_range.rb

@@ -0,0 +1,22 @@
+module Zermelo
+  module Filters
+    class IndexRange
+
+      attr_reader :start, :finish, :by_score
+
+      def initialize(start, finish, opts = {})
+        value_types = opts[:by_score] ? [Float, Date, Time, DateTime] : [Integer]
+        [start, finish].each do |v|
+          raise "Values must be #{value_types.join('/')}" unless v.nil? || value_types.any? {|vt| v.is_a?(vt)}
+        end
+        if !start.nil? && !finish.nil? && (start > finish)
+          raise "Start of range must be <= finish"
+        end
+        @start    = start
+        @finish   = finish
+        @by_score = opts[:by_score]
+      end
+
+    end
+  end
+end