zermelo 1.0.0

data/lib/zermelo/backends/redis_backend.rb

@@ -0,0 +1,281 @@
+require 'zermelo/backends/base'
+
+require 'zermelo/filters/redis_filter'
+require 'zermelo/locks/redis_lock'
+
+module Zermelo
+
+  module Backends
+
+    class RedisBackend
+
+      include Zermelo::Backends::Base
+
+      def generate_lock
+        Zermelo::Locks::RedisLock.new
+      end
+
+      def filter(ids_key, record)
+        Zermelo::Filters::RedisFilter.new(self, ids_key, record)
+      end
+
+      def get_multiple(*attr_keys)
+        attr_keys.inject({}) do |memo, attr_key|
+          redis_attr_key = key_to_redis_key(attr_key)
+
+          memo[attr_key.klass] ||= {}
+          memo[attr_key.klass][attr_key.id] ||= {}
+          memo[attr_key.klass][attr_key.id][attr_key.name.to_s] = if Zermelo::COLLECTION_TYPES.has_key?(attr_key.type)
+
+            case attr_key.type
+            when :list
+              if attr_key.accessor.nil?
+                Zermelo.redis.lrange(redis_attr_key, 0, -1)
+              else
+
+              end
+            when :set
+              if attr_key.accessor.nil?
+                Set.new( Zermelo.redis.smembers(redis_attr_key) )
+              else
+
+              end
+            when :hash
+              if attr_key.accessor.nil?
+                Zermelo.redis.hgetall(redis_attr_key)
+              else
+
+              end
+            when :sorted_set
+              # TODO should this be something that preserves order?
+              if attr_key.accessor.nil?
+                Set.new( Zermelo.redis.zrange(redis_attr_key, 0, -1) )
+              else
+
+              end
+            end
+
+          else
+            value = Zermelo.redis.hget(redis_attr_key, attr_key.name.to_s)
+
+            if value.nil?
+              nil
+            else
+              case attr_key.type
+              when :string
+                value.to_s
+              when :integer
+                value.to_i
+              when :float
+                value.to_f
+              when :timestamp
+                Time.at(value.to_f)
+              when :boolean
+                case value
+                when TrueClass
+                  true
+                when FalseClass
+                  false
+                when String
+                  'true'.eql?(value.downcase)
+                else
+                  nil
+                end
+              end
+            end
+          end
+          memo
+        end
+      end
+
+      def exists?(key)
+        Zermelo.redis.exists(key_to_redis_key(key))
+      end
+
+      def include?(key, id)
+        case key.type
+        when :set
+          Zermelo.redis.sismember(key_to_redis_key(key), id)
+        else
+          raise "Not implemented"
+        end
+      end
+
+      def begin_transaction
+        return false unless @transaction_redis.nil?
+        @transaction_redis = Zermelo.redis
+        @transaction_redis.multi
+        @changes = []
+        true
+      end
+
+      def commit_transaction
+        return false if @transaction_redis.nil?
+        apply_changes(@changes)
+        @transaction_redis.exec
+        @transaction_redis = nil
+        @changes = []
+        true
+      end
+
+      def abort_transaction
+        return false if @transaction_redis.nil?
+        @transaction_redis.discard
+        @transaction_redis = nil
+        @changes = []
+        true
+      end
+
+      # used by redis_filter
+      def key_to_redis_key(key)
+        obj = case key.object
+        when :attribute
+          'attrs'
+        when :association
+          'assocs'
+        when :index
+          'indices'
+        end
+
+        name = Zermelo::COLLECTION_TYPES.has_key?(key.type) ? ":#{key.name}" : ''
+
+        "#{key.klass}:#{key.id.nil? ? '' : key.id}:#{obj}#{name}"
+      end
+
+      private
+
+      def change(op, key, value = nil, key_to = nil)
+        ch = [op, key, value, key_to]
+        if @in_transaction
+          @changes << ch
+        else
+          apply_changes([ch])
+        end
+      end
+
+      def apply_changes(changes)
+        simple_attrs  = {}
+
+        purges = []
+
+        changes.each do |ch|
+          op     = ch[0]
+          key    = ch[1]
+          value  = ch[2]
+          key_to = ch[3]
+
+          # TODO check that collection types handle nil value for whole thing
+          if Zermelo::COLLECTION_TYPES.has_key?(key.type)
+
+            complex_attr_key = key_to_redis_key(key)
+
+            case op
+            when :add, :set
+              case key.type
+              when :list
+                Zermelo.redis.del(complex_attr_key) if :set.eql?(op)
+                Zermelo.redis.rpush(complex_attr_key, value)
+              when :set
+                Zermelo.redis.del(complex_attr_key) if :set.eql?(op)
+                case value
+                when Set
+                  Zermelo.redis.sadd(complex_attr_key, value.to_a) unless value.empty?
+                when Array
+                  Zermelo.redis.sadd(complex_attr_key, value) unless value.empty?
+                else
+                  Zermelo.redis.sadd(complex_attr_key, value)
+                end
+              when :hash
+                Zermelo.redis.del(complex_attr_key) if :set.eql?(op)
+                unless value.nil?
+                  kv = value.inject([]) do |memo, (k, v)|
+                    memo += [k, v]
+                    memo
+                  end
+                  Zermelo.redis.hmset(complex_attr_key, *kv)
+                end
+              when :sorted_set
+                Zermelo.redis.zadd(complex_attr_key, *value)
+              end
+            when :move
+              case key.type
+              when :set
+                Zermelo.redis.smove(complex_attr_key, key_to_redis_key(key_to), value)
+              when :list
+                # TODO would do via sort 'nosort', except for
+                # https://github.com/antirez/redis/issues/2079 -- instead,
+                # copy the workaround from redis_filter.rb
+                raise "Not yet implemented"
+              when :hash
+                values = value.to_a.flatten
+                Zermelo.redis.hdel(complex_attr_key, values)
+                Zermelo.redis.hset(key_to_redis_key(key_to), *values)
+              when :sorted_set
+                raise "Not yet implemented"
+              end
+            when :delete
+              case key.type
+              when :list
+                Zermelo.redis.lrem(complex_attr_key, value, 0)
+              when :set
+                Zermelo.redis.srem(complex_attr_key, value)
+              when :hash
+                Zermelo.redis.hdel(complex_attr_key, value)
+              when :sorted_set
+                Zermelo.redis.zrem(complex_attr_key, value)
+              end
+            when :clear
+              Zermelo.redis.del(complex_attr_key)
+            end
+
+          elsif :purge.eql?(op)
+            # TODO get keys for all assocs & indices, purge them too
+            purges << ["#{key.klass}:#{key.id}:attrs"]
+          else
+            simple_attr_key = key_to_redis_key(key)
+            simple_attrs[simple_attr_key] ||= {}
+
+            case op
+            when :set
+              simple_attrs[simple_attr_key][key.name] = if value.nil?
+                nil
+              else
+                case key.type
+                when :string, :integer
+                  value.to_s
+                when :float, :timestamp
+                  value.to_f
+                when :boolean
+                  (!!value).to_s
+                end
+              end
+            when :clear
+              simple_attrs[simple_attr_key][key.name] = nil
+            end
+          end
+        end
+
+        unless simple_attrs.empty?
+          simple_attrs.each_pair do |simple_attr_key, values|
+            hset = []
+            hdel = []
+            values.each_pair do |k, v|
+              if v.nil?
+                hdel << k
+              else
+                hset += [k, v]
+              end
+            end
+            Zermelo.redis.hmset(simple_attr_key, *hset) if hset.present?
+            Zermelo.redis.hdel(simple_attr_key, hdel) if hdel.present?
+          end
+        end
+
+        purges.each {|purge_key | Zermelo.redis.del(purge_key) }
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/filters/base.rb

@@ -0,0 +1,235 @@
+require 'active_support/concern'
+
+require 'zermelo/records/errors'
+
+require 'zermelo/filters/steps/diff_range_step'
+require 'zermelo/filters/steps/diff_step'
+require 'zermelo/filters/steps/intersect_range_step'
+require 'zermelo/filters/steps/intersect_step'
+require 'zermelo/filters/steps/limit_step'
+require 'zermelo/filters/steps/offset_step'
+require 'zermelo/filters/steps/sort_step'
+require 'zermelo/filters/steps/union_range_step'
+require 'zermelo/filters/steps/union_step'
+
+module Zermelo
+
+  module Filters
+
+    module Base
+
+      extend ActiveSupport::Concern
+
+      attr_reader :backend, :steps
+
+      # initial set         a Zermelo::Record::Key object
+      # associated_class    the class of the result record
+      def initialize(data_backend, initial_set, associated_class, ancestor = nil, step = nil)
+        @backend          = data_backend
+        @initial_set      = initial_set
+        @associated_class = associated_class
+        @steps            = ancestor.nil? ? [] : ancestor.steps.dup
+        @steps << step unless step.nil?
+      end
+
+      # TODO each step type have class methods list its acceptable input types, and
+      # have a class method giving its output type
+
+      def intersect(attrs = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::IntersectStep.new({}, attrs))
+      end
+
+      def union(attrs = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::UnionStep.new({}, attrs))
+      end
+
+      def diff(attrs = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::DiffStep.new({}, attrs))
+      end
+
+      def sort(keys, opts = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::SortStep.new({:keys => keys,
+            :desc => opts[:desc], :limit => opts[:limit],
+            :offset => opts[:offset]}, {})
+          )
+      end
+
+      def limit(amount)
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::LimitStep.new({:amount => amount}, {}))
+      end
+
+      def offset(amount)
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::OffsetStep.new({:amount => amount}, {}))
+      end
+
+      def intersect_range(start, finish, attrs_opts = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::IntersectRangeStep.new(
+            {:start => start, :finish => finish,
+             :desc => attrs_opts.delete(:desc),
+             :by_score => attrs_opts.delete(:by_score)},
+            attrs_opts)
+          )
+      end
+
+      def union_range(start, finish, attrs_opts = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::UnionRangeStep.new(
+            {:start => start, :finish => finish,
+             :desc => attrs_opts.delete(:desc),
+             :by_score => attrs_opts.delete(:by_score)},
+            attrs_opts)
+          )
+      end
+
+      def diff_range(start, finish, attrs_opts = {})
+        self.class.new(@backend, @initial_set, @associated_class, self,
+          ::Zermelo::Filters::Steps::DiffRangeStep.new(
+            {:start => start, :finish => finish,
+             :desc => attrs_opts.delete(:desc),
+             :by_score => attrs_opts.delete(:by_score)},
+            attrs_opts)
+          )
+      end
+
+      # step users
+      def exists?(e_id)
+        lock(false) { _exists?(e_id) }
+      end
+
+      def find_by_id(f_id)
+        lock { _find_by_id(f_id) }
+      end
+
+      def find_by_id!(f_id)
+        ret = lock { _find_by_id(f_id) }
+        raise ::Zermelo::Records::Errors::RecordNotFound.new(@associated_class, f_id) if ret.nil?
+        ret
+      end
+
+      def find_by_ids(*f_ids)
+        lock { f_ids.collect {|f_id| _find_by_id(f_id) } }
+      end
+
+      def find_by_ids!(*f_ids)
+        ret = lock { f_ids.collect {|f_id| _find_by_id(f_id) } }
+        if ret.any? {|r| r.nil? }
+          raise ::Zermelo::Records::Errors::RecordsNotFound.new(@associated_class, f_ids - ret.compact.map(&:id))
+        end
+        ret
+      end
+
+      def ids
+        lock(false) { _ids }
+      end
+
+      def count
+        lock(false) { _count }
+      end
+
+      def empty?
+        lock(false) { _count == 0 }
+      end
+
+      def all
+        lock { _all }
+      end
+
+      # NB makes no sense to apply this without order clause
+      def page(num, opts = {})
+        ret = nil
+        per_page = opts[:per_page].to_i || 20
+        if (num > 0) && (per_page > 0)
+          lock do
+            start  = per_page * (num - 1)
+            finish = start + (per_page - 1)
+            @steps += [Zermelo::Filters::Steps::OffsetStep.new({:amount => start},    {}),
+                       Zermelo::Filters::Steps::LimitStep.new({:amount => per_page}, {})]
+            page_ids = _ids
+            ret = page_ids.collect {|f_id| _load(f_id)} unless page_ids.nil?
+          end
+        end
+        ret || []
+      end
+
+      def collect(&block)
+        lock { _ids.collect {|id| block.call(_load(id))} }
+      end
+      alias_method :map, :collect
+
+      def each(&block)
+        lock { _ids.each {|id| block.call(_load(id)) } }
+      end
+
+      def select(&block)
+        lock { _all.select {|obj| block.call(obj) } }
+      end
+      alias_method :find_all, :select
+
+      def reject(&block)
+        lock { _all.reject {|obj| block.call(obj)} }
+      end
+
+      def destroy_all
+        lock(*@associated_class.send(:associated_classes)) do
+          _all.each {|r| r.destroy }
+        end
+      end
+
+      def associated_ids_for(name, options = {})
+        klass = @associated_class.send(:with_association_data, name.to_sym) do |data|
+          data.type_klass
+        end
+
+        lock {
+          case klass.name
+          when ::Zermelo::Associations::BelongsTo.name
+            klass.send(:associated_ids_for, @backend,
+              @associated_class.send(:class_key), name,
+              options[:inversed].is_a?(TrueClass), *_ids)
+          else
+            klass.send(:associated_ids_for, @backend,
+              @associated_class.send(:class_key), name, *_ids)
+          end
+        }
+      end
+
+      protected
+
+      def lock(when_steps_empty = true, *klasses, &block)
+        return(block.call) if !when_steps_empty && @steps.empty?
+        klasses += [@associated_class] if !klasses.include?(@associated_class)
+        @backend.lock(*klasses, &block)
+      end
+
+      private
+
+      def _find_by_id(id)
+        if !id.nil? && _exists?(id)
+          _load(id.to_s)
+        else
+          nil
+        end
+      end
+
+      def _load(id)
+        object = @associated_class.new
+        object.load(id)
+        object
+      end
+
+      def _all
+        _ids.map {|id| _load(id) }
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/filters/influxdb_filter.rb

@@ -0,0 +1,162 @@
+require 'zermelo/filters/base'
+
+module Zermelo
+  module Filters
+    class InfluxDBFilter
+
+      include Zermelo::Filters::Base
+
+      private
+
+      def _exists?(id)
+        return if id.nil?
+        @steps << Zermelo::Filters::Steps::IntersectStep.new({}, {:id => id})
+        resolve_steps(:count) > 0
+      end
+
+      def lock(when_steps_empty = true, *klasses, &block)
+        # no-op
+        block.call
+      end
+
+      def _ids
+        resolve_steps(:ids)
+      end
+
+      def _count
+        resolve_steps(:count)
+      end
+
+      def resolve_step(step)
+        query = ''
+
+        options   = step.options || {}
+        values    = step.attributes
+
+        case step
+        when Zermelo::Filters::Steps::IntersectStep,
+             Zermelo::Filters::Steps::UnionStep
+
+          query += values.collect {|k, v|
+            op, value = case v
+            when String
+              ["=~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+            else
+              ["=",  "'#{v}'"]
+            end
+
+           "#{k} #{op} #{value}"
+          }.join(' AND ')
+
+        when Zermelo::Filters::Steps::DiffStep
+
+          query += values.collect {|k, v|
+            op, value = case v
+            when String
+              ["!~", "/^#{Regexp.escape(v).gsub(/\\\\/, "\\")}$/"]
+            else
+              ["!=",  "'#{v}'"]
+            end
+
+            "#{k} #{op} #{value}"
+          }.join(' AND ')
+        else
+          raise "Unhandled filter operation '#{step_type}'"
+        end
+
+        query
+      end
+
+      def escaped_id(id)
+        if id.is_a?(Numeric)
+          id
+        else
+          "'" + id.gsub(/'/, "\\'").gsub(/\\/, "\\\\'") + "'"
+        end
+      end
+
+      def resolve_steps(result_type)
+        query = case result_type
+        when :ids
+          "SELECT id FROM /#{@associated_class.send(:class_key)}\\/.*/"
+        when :count
+          "SELECT COUNT(id) FROM /#{@associated_class.send(:class_key)}\\/.*/"
+        end
+
+        unless @initial_set.id.nil?
+          query += ' WHERE '
+
+          ii_query = "SELECT #{@initial_set.name} FROM \"#{@initial_set.klass}/#{@initial_set.id}\" " +
+            "LIMIT 1"
+
+          begin
+            initial_id_data =
+              Zermelo.influxdb.query(ii_query)["#{@initial_set.klass}/#{@initial_set.id}"]
+          rescue InfluxDB::Error => ide
+            raise unless
+              /^Field #{@initial_set.name} doesn't exist in series #{@initial_set.klass}\/#{@initial_set.id}$/ === ide.message
+
+            initial_id_data = nil
+          end
+
+          return [] if initial_id_data.nil?
+
+          inital_ids = initial_id_data.first[@initial_set.name]
+
+          if inital_ids.nil? || inital_ids.empty?
+            # make it impossible for the query to return anything
+            query += '(1 = 0)'
+          else
+            query += '((' + inital_ids.collect {|id|
+              "id = #{escaped_id(id)}"
+            }.join(') OR (') + '))'
+          end
+        end
+
+        unless @steps.empty?
+          query += (@initial_set.id.nil? ? ' WHERE ' : ' AND ') +
+                   ('(' * @steps.size)
+
+          @steps.each_with_index do |step, idx|
+            if idx > 0
+              case step
+              when Zermelo::Filters::Steps::IntersectStep,
+                   Zermelo::Filters::Steps::DiffStep
+                query += ' AND '
+              when Zermelo::Filters::Steps::UnionStep
+                query += ' OR '
+              else
+                raise "Unhandled filter operation '#{step.class.name}'"
+              end
+            end
+
+            query += resolve_step(step)
+
+            query += ")"
+          end
+        end
+
+        query += " LIMIT 1"
+
+        begin
+          result = Zermelo.influxdb.query(query)
+        rescue InfluxDB::Error => ide
+          raise unless /^Couldn't look up columns$/ === ide.message
+          result = {}
+        end
+
+        data_keys = result.keys.select {|k| k =~ /^#{@associated_class.send(:class_key)}\// }
+
+        case result_type
+        when :ids
+          data_keys.empty? ? [] : data_keys.collect {|k| k =~ /^#{@associated_class.send(:class_key)}\/(.+)$/; $1 }
+        when :count
+          data_keys.empty? ?  0 : data_keys.inject(0) do |memo, k|
+            memo += result[k].first['count']
+            memo
+          end
+        end
+      end
+    end
+  end
+end