zermelo 1.0.0

data/lib/zermelo/associations/has_one.rb

@@ -0,0 +1,109 @@
+module Zermelo
+  module Associations
+    class HasOne
+
+      def initialize(parent, name)
+        @parent = parent
+
+        @backend = parent.send(:backend)
+
+        # TODO would be better as a 'has_one' hash, a bit like belongs_to
+        @record_id_key = Zermelo::Records::Key.new(
+          :klass  => parent.class.send(:class_key),
+          :id     => parent.id,
+          :name   => "#{name}_id",
+          :type   => :string,
+          :object => :association
+        )
+
+        parent.class.send(:with_association_data, name.to_sym) do |data|
+          @associated_class = data.data_klass
+          @lock_klasses     = [data.data_klass] + data.related_klasses
+          @inverse          = data.inverse
+          @callbacks        = data.callbacks
+        end
+      end
+
+      def value
+        @parent.class.lock(*@associated_class) do
+          if id = @backend.get(@record_id_key)
+            @associated_class.send(:load, id)
+          else
+            nil
+          end
+        end
+      end
+
+      def value=(record)
+        if record.nil?
+          @parent.class.lock(*@lock_klasses) do
+           id = @backend.get(@record_id_key)
+           unless id.nil?
+             r = @associated_class.send(:load, id)
+             unless r.nil?
+               bc = @callbacks[:before_clear]
+               if bc.nil? || !@parent.respond_to?(bc) || !@parent.send(bc, r).is_a?(FalseClass)
+                  delete_without_lock(r)
+                  ac = @callbacks[:after_clear]
+                  @parent.send(ac, r) if !ac.nil? && @parent.respond_to?(ac)
+                end
+              end
+            end
+          end
+        else
+          raise 'Invalid record class' unless record.is_a?(@associated_class)
+          raise 'Record must have been saved' unless record.persisted?
+          @parent.class.lock(*@lock_klasses) do
+            bs = @callbacks[:before_set]
+            if bs.nil? || !@parent.respond_to?(bs) || !@parent.send(bs, r).is_a?(FalseClass)
+              unless @inverse.nil?
+                @associated_class.send(:load, record.id).send("#{@inverse}=", @parent)
+              end
+
+              new_txn = @backend.begin_transaction
+              @backend.set(@record_id_key, record.id)
+              @backend.commit_transaction if new_txn
+              as = @callbacks[:after_set]
+              @parent.send(as, record) if !as.nil? && @parent.respond_to?(as)
+            end
+          end
+        end
+      end
+
+      private
+
+      def delete_without_lock(record)
+        unless @inverse.nil?
+          record.send("#{@inverse}=", nil)
+        end
+        new_txn = @backend.begin_transaction
+        @backend.clear(@record_id_key)
+        @backend.commit_transaction if new_txn
+      end
+
+      # associated will be a belongs_to; on_remove already runs inside lock and transaction
+      def on_remove
+        unless @inverse.nil?
+          if record_id = @backend.get(@record_id_key)
+            @associated_class.send(:load, record_id).send("#{@inverse}=", nil)
+          end
+        end
+        @backend.clear(@record_id_key)
+      end
+
+      def self.associated_ids_for(backend, class_key, name, *these_ids)
+        these_ids.each_with_object({}) do |this_id, memo|
+          key = Zermelo::Records::Key.new(
+            :klass  => class_key,
+            :id     => this_id,
+            :name   => "#{name}_id",
+            :type   => :string,
+            :object => :association
+          )
+          memo[this_id] = backend.get(key)
+        end
+      end
+
+    end
+  end
+end

data/lib/zermelo/associations/has_sorted_set.rb

@@ -0,0 +1,124 @@
+require 'forwardable'
+
+module Zermelo
+  module Associations
+    class HasSortedSet
+
+      extend Forwardable
+
+      def_delegators :filter, :intersect, :union, :diff, :sort,
+                       :intersect_range, :union_range, :diff_range,
+                       :find_by_id, :find_by_ids, :find_by_id!, :find_by_ids!,
+                       :all, :each, :collect, :map,
+                       :select, :find_all, :reject, :destroy_all,
+                       :ids, :count, :empty?, :exists?,
+                       :first, :last,
+                       :associated_ids_for
+
+      def initialize(parent, name)
+        @parent = parent
+
+        @backend = parent.send(:backend)
+
+        @record_ids_key = Zermelo::Records::Key.new(
+          :klass  => parent.class.send(:class_key),
+          :id     => parent.id,
+          :name   => "#{name}_ids",
+          :type   => :sorted_set,
+          :object => :association
+        )
+
+        parent.class.send(:with_association_data, name.to_sym) do |data|
+          @associated_class = data.data_klass
+          @lock_klasses     = [data.data_klass] + data.related_klasses
+          @inverse          = data.inverse
+          @sort_key         = data.sort_key
+          @callbacks        = data.callbacks
+        end
+      end
+
+      def <<(record)
+        add(record)
+        self  # for << 'a' << 'b'
+      end
+
+      # TODO collect all scores/ids and do a single zadd/single hmset
+      def add(*records)
+        raise 'No records to add' if records.empty?
+        raise 'Invalid record class' if records.any? {|r| !r.is_a?(@associated_class)}
+        raise 'Record(s) must have been saved' unless records.all? {|r| r.persisted?}
+        @parent.class.lock(*@lock_klasses) do
+          ba = @callbacks[:before_add]
+          if ba.nil? || !@parent.respond_to?(ba) || !@parent.send(ba, *records).is_a?(FalseClass)
+            unless @inverse.nil?
+              records.each do |record|
+                @associated_class.send(:load, record.id).send("#{@inverse}=", @parent)
+              end
+            end
+
+            new_txn = @backend.begin_transaction
+            @backend.add(@record_ids_key, (records.map {|r| [r.send(@sort_key.to_sym).to_f, r.id]}.flatten))
+            @backend.commit_transaction if new_txn
+            aa = @callbacks[:after_add]
+            @parent.send(aa, *records) if !aa.nil? && @parent.respond_to?(aa)
+          end
+        end
+      end
+
+      def delete(*records)
+        raise 'No records to delete' if records.empty?
+        raise 'Invalid record class' if records.any? {|r| !r.is_a?(@associated_class)}
+        raise 'Record(s) must have been saved' unless records.all? {|r| r.persisted?}
+        @parent.class.lock(*@lock_klasses) do
+          br = @callbacks[:before_remove]
+          if br.nil? || !@parent.respond_to?(br) || !@parent.send(br, *records).is_a?(FalseClass)
+            unless @inverse.nil?
+              records.each do |record|
+                @associated_class.send(:load, record.id).send("#{@inverse}=", nil)
+              end
+            end
+
+            new_txn = @backend.begin_transaction
+            @backend.delete(@record_ids_key, records.map(&:id))
+            @backend.commit_transaction if new_txn
+            ar = @callbacks[:after_remove]
+            @parent.send(ar, *records) if !ar.nil? && @parent.respond_to?(ar)
+          end
+        end
+      end
+
+      private
+
+      # associated will be a belongs_to; on remove already runs inside a lock and transaction
+      def on_remove
+        unless @inverse.nil?
+          self.ids.each do |record_id|
+            # clear the belongs_to inverse value with this @parent.id
+            @associated_class.send(:load, record_id).send("#{@inverse}=", nil)
+          end
+        end
+        @backend.clear(@record_ids_key)
+      end
+
+      # creates a new filter class each time it's called, to store the
+      # state for this particular filter chain
+      def filter
+        @backend.filter(@record_ids_key, @associated_class)
+      end
+
+      def self.associated_ids_for(backend, class_key, name, *these_ids)
+        these_ids.each_with_object({}) do |this_id, memo|
+          key = Zermelo::Records::Key.new(
+            :klass  => class_key,
+            :id     => this_id,
+            :name   => "#{name}_ids",
+            :type   => :sorted_set,
+            :object => :association
+          )
+          memo[this_id] = backend.get(key)
+        end
+      end
+
+    end
+  end
+end

data/lib/zermelo/associations/index.rb

@@ -0,0 +1,50 @@
+# NB index instances are all internal to zermelo, not user-accessible
+
+module Zermelo
+  module Associations
+    class Index
+
+      def initialize(parent_klass, name)
+        @parent_klass   = parent_klass
+        @attribute_name = name
+
+        @backend   = parent_klass.send(:backend)
+        @class_key = parent_klass.send(:class_key)
+
+        @indexers = {}
+
+        parent_klass.send(:with_index_data, name.to_sym) do |data|
+          @attribute_type = data.type
+        end
+      end
+
+      def delete_id(id, value)
+        return unless indexer = key(value)
+        @backend.delete(indexer, id)
+      end
+
+      def add_id(id, value)
+        return unless indexer = key(value)
+        @backend.add(indexer, id)
+      end
+
+      def move_id(id, value_from, indexer_to, value_to)
+        return unless indexer = key(value_from)
+        @backend.move(indexer, id, indexer_to.key(value_to))
+      end
+
+      def key(value)
+        index_keys = @backend.index_keys(@attribute_type, value)
+        raise "Can't index '#{@value}' (#{@attribute_type}" if index_keys.nil?
+
+        @indexers[index_keys.join(":")] ||= Zermelo::Records::Key.new(
+          :klass  => @class_key,
+          :name   => "by_#{@attribute_name}:#{index_keys.join(':')}",
+          :type   => :set,
+          :object => :index
+        )
+      end
+
+    end
+  end
+end

data/lib/zermelo/associations/index_data.rb

@@ -0,0 +1,18 @@
+module Zermelo
+  module Associations
+    class IndexData
+      attr_writer  :data_klass_name
+      attr_accessor :name, :type, :index_klass
+
+      def initialize(opts = {})
+        [:name, :type, :index_klass].each do |a|
+          send("#{a}=".to_sym, opts[a])
+        end
+      end
+
+      def data_klass
+        @data_klass ||= @data_klass_name.constantize
+      end
+    end
+  end
+end

data/lib/zermelo/associations/unique_index.rb

@@ -0,0 +1,44 @@
+# NB index instances are all internal to zermelo, not user-accessible
+
+module Zermelo
+  module Associations
+    class UniqueIndex
+
+      def initialize(parent_klass, name)
+        @parent_klass   = parent_klass
+        @attribute_name = name
+
+        @backend   = parent_klass.send(:backend)
+        @class_key = parent_klass.send(:class_key)
+
+        @indexers = {}
+
+        parent_klass.send(:with_index_data, name.to_sym) do |data|
+          @attribute_type = data.type
+        end
+      end
+
+      def delete_id(id, value)
+        @backend.delete(key, @backend.index_keys(@attribute_type, value).join(':'))
+      end
+
+      def add_id(id, value)
+        @backend.add(key, @backend.index_keys(@attribute_type, value).join(':') => id)
+      end
+
+      def move_id(id, value_from, indexer_to, value_to)
+        @backend.move(key, {@backend.index_keys(@attribute_type, value_to).join(':') => id}, indexer_to.key)
+      end
+
+      def key
+        @indexer ||= Zermelo::Records::Key.new(
+          :klass  => @class_key,
+          :name   => "by_#{@attribute_name}",
+          :type   => :hash,
+          :object => :index
+        )
+      end
+
+    end
+  end
+end

data/lib/zermelo/backends/base.rb

@@ -0,0 +1,115 @@
+require 'active_support/concern'
+
+require 'zermelo/locks/no_lock'
+
+module Zermelo
+
+  module Backends
+
+    module Base
+
+      extend ActiveSupport::Concern
+
+      def escape_key_name(name)
+        name.gsub(/%/, '%%').gsub(/ /, '%20').gsub(/:/, '%3A')
+      end
+
+      def unescape_key_name(name)
+        name.gsub(/%3A/, ':').gsub(/%20/, ' ').gsub(/%%/, '%')
+      end
+
+      def index_keys(type, value)
+        return ["null", "null"] if value.nil?
+
+        case type
+        when :string
+          ["string", escape_key_name(value)]
+        when :integer
+          ["integer", escape_key_name(value.to_s)]
+        when :float
+          ["float", escape_key_name(value.to_s)]
+        when :timestamp
+          case value
+          when Integer
+            ["timestamp", escape_key_name(value.to_s)]
+          when Time, DateTime
+            ["timestamp", escape_key_name(value.to_i.to_s)]
+          end
+        when :boolean
+          case value
+          when TrueClass
+            ["boolean", "true"]
+          when FalseClass
+            ["boolean", "false"]
+          end
+        end
+      end
+
+      # for hashes, lists, sets
+      def add(key, value)
+        change(:add, key, value)
+      end
+
+      def delete(key, value)
+        change(:delete, key, value)
+      end
+
+      def move(key, value, key_to)
+        change(:move, key, value, key_to)
+      end
+
+      def clear(key)
+        change(:clear, key)
+      end
+
+      # works for both simple and complex types (i.e. strings, numbers, booleans,
+      #  hashes, lists, sets)
+      def set(key, value)
+        change(:set, key, value)
+      end
+
+      def purge(key)
+        change(:purge, key)
+      end
+
+      def get(attr_key)
+        get_multiple(attr_key)[attr_key.klass][attr_key.id][attr_key.name.to_s]
+      end
+
+      def lock(*klasses, &block)
+        ret = nil
+        # doesn't handle re-entrant case for influxdb, which has no locking yet
+        locking = Thread.current[:zermelo_locking]
+        if locking.nil?
+          lock_proc = proc do
+            begin
+              Thread.current[:zermelo_locking] = klasses
+              ret = block.call
+            ensure
+              Thread.current[:zermelo_locking] = nil
+            end
+          end
+
+          lock_klass = case self
+          when Zermelo::Backends::RedisBackend
+            Zermelo::Locks::RedisLock
+          else
+            Zermelo::Locks::NoLock
+          end
+
+          lock_klass.new.lock(*klasses, &lock_proc)
+        else
+          # accepts any subset of 'locking'
+          unless (klasses - locking).empty?
+            raise "Currently locking #{locking.map(&:name)}, cannot lock different set #{klasses.map(&:name)}"
+          end
+          ret = block.call
+        end
+        ret
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/backends/influxdb_backend.rb

@@ -0,0 +1,178 @@
+require 'zermelo/backends/base'
+
+require 'zermelo/filters/influxdb_filter'
+
+# NB influxdb doesn't support individually addressable deletes, so
+# this backend only works to write new records
+# (it could just write the new state of the record, and query id by newest limit 1,
+#  but for the moment, YAGNI)
+
+module Zermelo
+
+  module Backends
+
+    class InfluxDBBackend
+
+      include Zermelo::Backends::Base
+
+      def filter(ids_key, record)
+        Zermelo::Filters::InfluxDBFilter.new(self, ids_key, record)
+      end
+
+      # TODO get filter calling this instead of using same logic
+      def exists?(key)
+        return if key.id.nil?
+        Zermelo.influxdb.query("SELECT id FROM /#{key.klass}\\/.*/ LIMIT 1").size > 0
+      end
+
+      # nb: does lots of queries, should batch, but ensuring single operations are correct
+      # for now
+      def get_multiple(*attr_keys)
+        attr_keys.inject({}) do |memo, attr_key|
+          begin
+          records = Zermelo.influxdb.query("SELECT #{attr_key.name} FROM " +
+            "\"#{attr_key.klass}/#{attr_key.id}\" LIMIT 1")["#{attr_key.klass}/#{attr_key.id}"]
+          rescue InfluxDB::Error => ide
+            raise unless
+              /^Field #{attr_key.name} doesn't exist in series #{attr_key.klass}\/#{attr_key.id}$/ === ide.message
+
+            records = []
+          end
+          value = (records && !records.empty?) ? records.first[attr_key.name.to_s] : nil
+
+          memo[attr_key.klass] ||= {}
+          memo[attr_key.klass][attr_key.id] ||= {}
+
+          memo[attr_key.klass][attr_key.id][attr_key.name.to_s] = if value.nil?
+            nil
+          else
+
+            case attr_key.type
+            when :string
+              value.to_s
+            when :integer
+              value.to_i
+            when :float
+              value.to_f
+            when :timestamp
+              Time.at(value.to_f)
+            when :boolean
+              case value
+              when TrueClass
+                true
+              when FalseClass
+                false
+              when String
+                'true'.eql?(value.downcase)
+              else
+                nil
+              end
+            when :list, :hash
+              value
+            when :set
+              Set.new(value)
+            end
+          end
+          memo
+        end
+      end
+
+      def begin_transaction
+        return false if @in_transaction
+        @in_transaction = true
+        @changes = []
+      end
+
+      def commit_transaction
+        return false unless @in_transaction
+        apply_changes(@changes)
+        @in_transaction = false
+        @changes = []
+      end
+
+      def abort_transaction
+        return false unless @in_transaction
+        @in_transaction = false
+        @changes = []
+      end
+
+      private
+
+      def change(op, key, value = nil)
+        ch = [op, key, value]
+        if @in_transaction
+          @changes << ch
+        else
+          apply_changes([ch])
+        end
+      end
+
+      # composite all new changes into records, and then into influxdb
+      # query statements
+      def apply_changes(changes)
+        records = {}
+
+        purges = []
+
+        changes.each do |ch|
+          op    = ch[0]
+          key   = ch[1]
+          value = ch[2]
+
+          next if key.id.nil?
+
+          records[key.klass]         ||= {}
+          records[key.klass][key.id] ||= {}
+
+          records[key.klass][key.id][key.name] = case op
+          when :set
+            case key.type
+            when :string, :integer
+              value.to_s
+            when :timestamp
+              value.to_f
+            when :boolean
+              (!!value).to_s
+            when :list, :hash
+              value
+            when :set
+              value.to_a
+            end
+          when :add
+            case key.type
+            when :list, :hash
+              value
+            when :set
+              value.to_a
+            end
+          when :purge
+            purges << "\"#{key.klass}/#{key.id}\""
+          end
+
+        end
+
+        records.each_pair do |klass, klass_records|
+          klass_records.each_pair do |id, data|
+            begin
+              prior = Zermelo.influxdb.query("SELECT * FROM \"#{klass}/#{id}\" LIMIT 1")["#{klass}/#{id}"]
+            rescue InfluxDB::Error => ide
+              raise unless
+                (/^Couldn't look up columns for series: #{klass}\/#{id}$/ === ide.message) ||
+                (/^Couldn't look up columns$/ === ide.message) ||
+                (/^Couldn't find series: #{klass}\/#{id}$/ === ide.message)
+
+              prior = nil
+            end
+            record = prior.nil? ? {} : prior.first.delete_if {|k,v| ["time", "sequence_number"].include?(k) }
+            Zermelo.influxdb.write_point("#{klass}/#{id}", record.merge(data).merge('id' => id))
+          end
+        end
+
+        purges.each {|purge| Zermelo.influxdb.query("DROP SERIES #{purge}") }
+      end
+
+    end
+
+  end
+
+end