zermelo 1.0.0

data/lib/zermelo/filters/steps/sort_step.rb

@@ -0,0 +1,17 @@
+require 'zermelo/filters/steps/base_step'
+
+module Zermelo
+  module Filters
+    class Steps
+      class SortStep < Zermelo::Filters::Steps::BaseStep
+        def self.accepted_types
+          [:set, :sorted_set]
+        end
+
+        def self.returns_type
+          :list
+        end
+      end
+    end
+  end
+end

data/lib/zermelo/filters/steps/union_range_step.rb

@@ -0,0 +1,17 @@
+require 'zermelo/filters/steps/base_step'
+
+module Zermelo
+  module Filters
+    class Steps
+      class UnionRangeStep < Zermelo::Filters::Steps::BaseStep
+        def self.accepted_types
+          [:sorted_set]
+        end
+
+        def self.returns_type
+          :sorted_set
+        end
+      end
+    end
+  end
+end

data/lib/zermelo/filters/steps/union_step.rb

@@ -0,0 +1,17 @@
+require 'zermelo/filters/steps/base_step'
+
+module Zermelo
+  module Filters
+    class Steps
+      class UnionStep < Zermelo::Filters::Steps::BaseStep
+        def self.accepted_types
+          [:set, :sorted_set]
+        end
+
+        def self.returns_type
+          :set
+        end
+      end
+    end
+  end
+end

data/lib/zermelo/locks/no_lock.rb

@@ -0,0 +1,16 @@
+
+module Zermelo
+
+  module Locks
+
+    class NoLock
+
+      def lock(*record_klasses, &block)
+        yield
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/locks/redis_lock.rb

@@ -0,0 +1,221 @@
+
+module Zermelo
+
+  class LockNotAcquired < StandardError; end
+
+  module Locks
+
+    class RedisLock
+
+      # Adapted from https://github.com/mlanett/redis-lock,
+      # now covers locking multiple keys at once
+
+      attr_accessor :expires_at, :life, :sleep_in_ms
+
+      def initialize
+        @owner_value = Thread.current.object_id
+        @life        = 60
+        @timeout     = 10
+        @sleep_in_ms = 125
+      end
+
+      def lock(*record_klasses, &block)
+        @keys = record_klasses.map{|k| k.send(:class_key) }.sort.map{|k| "#{k}::lock" }
+        do_lock_with_timeout(@timeout) or raise Zermelo::LockNotAcquired.new(@keys.join(", "))
+        result = true
+        if block
+          begin
+            result = (block.arity == 1) ? block.call(self) : block.call
+          # rescue Exception => e
+          #   puts e.message
+          #   puts e.backtrace.join("\n")
+          #   raise e
+          ensure
+            release_lock
+          end
+        end
+        result
+      end
+
+      def extend_life( new_life )
+        do_extend( new_life ) or raise Zermelo::LockNotAcquired.new(@keys.join(", "))
+      end
+
+      def unlock
+        release_lock
+      end
+
+      private
+
+      def full_keys
+        @full_keys ||= @keys.map {|k| ["#{k}:owner", "#{k}:expiry"] }.flatten
+      end
+
+      def owner_keys
+        @owner_keys ||= @keys.map {|k| "#{k}:owner" }
+      end
+
+      def expiry_keys
+        @expiry_keys ||= @keys.map {|k| "#{k}:expiry" }
+      end
+
+      def do_lock_with_timeout( timeout )
+        locked = false
+        with_timeout(timeout) { locked = do_lock }
+        locked
+      end
+
+      # @returns true if locked, false otherwise
+      def do_lock( tries = 3 )
+        # We need to set both owner and expire at the same time
+        # If the existing lock is stale, we delete it and try again once
+
+        locked = nil
+
+        loop do
+          new_xval = Time.now.to_i + @life
+
+          lock_keyvals = @keys.map {|k| ["#{k}:owner",  @owner_value,
+                                         "#{k}:expiry", new_xval] }.flatten
+
+          result = Zermelo.redis.msetnx(*lock_keyvals)
+
+          if [1, true].include?(result)
+            # log :debug, "do_lock() success"
+            @expires_at = new_xval
+            locked = true
+            break
+          else
+            # log :debug, "do_lock() failed"
+            # consider the possibility that this lock is stale
+            tries -= 1
+            next if tries > 0 && stale_key?
+            locked = false
+            break
+          end
+        end
+        locked
+      end
+
+      def do_extend( new_life )
+        # We use watch and a transaction to ensure we only change a lock we own
+        # The transaction fails if the watched variable changed
+        # Use my_owner = oval to make testing easier.
+        new_xval = Time.now.to_i + new_life
+        extended = false
+        with_watch( *owner_keys  ) do
+          owners = Zermelo.redis.mget( *owner_keys )
+          if owners == ([@owner_value.to_s] * owner_keys.size)
+            result = Zermelo.redis.multi do |multi|
+              multi.mset( *(expiry_keys.zip( [new_xval] * expiry_keys.size)) )
+            end
+            if result == ['OK']
+              # log :debug, "do_extend() success"
+              @expires_at = new_xval
+              extended = true
+            end
+          end
+        end
+        extended
+      end
+
+      # Only actually deletes it if we own it.
+      # There may be strange cases where we fail to delete it, in which case expiration will solve the problem.
+      def release_lock
+        released = false
+        with_watch( *full_keys ) do
+          owners = Zermelo.redis.mget( *owner_keys )
+          if owners == ([@owner_value.to_s] * owner_keys.size)
+            result = Zermelo.redis.multi do |multi|
+              multi.del(*full_keys)
+            end
+            if result && (result.size == 1) && (result.first == full_keys.size)
+              released = true
+            end
+          end
+        end
+        released
+      end
+
+      def stale_key?
+        # Check if expiration exists and is it stale?
+        # If so, delete it.
+        # watch() all keys so we can detect if they change while we do this
+        # multi() will fail if keys have changed after watch()
+        # Thus, we snapshot consistency at the time of watch()
+        # Note: inside a watch() we get one and only one multi()
+        now = Time.now.to_i
+        stale = false
+        with_watch( *full_keys ) do
+
+          owners_expires = Zermelo.redis.mget(full_keys)
+
+          if owners_expires.each_slice(2).all? {|owner, expire| is_deletable?( owner, expire, now)}
+            result = Zermelo.redis.multi do |multi|
+              multi.del(*full_keys)
+            end
+            # If anything changed then multi() fails and returns nil
+            if result && (result.size == 1) && (result.first == owner_keys.size)
+              # log :info, "Deleted stale key from #{owner}"
+              stale = true
+            end
+          end
+        end # watch
+        stale
+      end
+
+      def locked?
+        now = Time.now.to_i
+        owners_expires = Zermelo.redis.mget(full_keys)
+        owners_expires && (owners_expires.size == (@keys.size * 2)) &&
+          owners_expires.each_slice(2).all? {|owner, expiration| is_locked?(owner, expiration, now)}
+      end
+
+      # returns true if the lock exists and is owned by the given owner
+      def is_locked?(owner, expiration, now)
+        (owner == @owner_value) && ! is_deletable?(owner, expiration, now)
+      end
+
+      # returns true if this is a broken or expired lock
+      def is_deletable?( owner, expiration, now)
+        expiration = expiration.to_i
+        (owner || (expiration > 0)) && (!owner || (expiration < now))
+      end
+
+      def with_watch( *args, &block )
+        Zermelo.redis.watch( *args )
+        begin
+          block.call
+        ensure
+          Zermelo.redis.unwatch
+        end
+      end
+
+      # Calls block until it returns true or times out.
+      # @param block should return true if successful, false otherwise
+      # @returns true if successful, false otherwise
+      def with_timeout( timeout, &block )
+        expire = Time.now + timeout.to_f
+        sleepy = @sleep_in_ms / 1000.to_f()
+        # this looks inelegant compared to while Time.now < expire, but does not oversleep
+        ret = nil
+        loop do
+          if block.call
+            ret = true
+            break
+          end
+          if (Time.now + sleepy) > expire
+            ret = false
+            break
+          end
+          sleep(sleepy)
+          # might like a different strategy, but general goal is not use 100% cpu while contending for a lock.
+        end
+        ret
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/records/base.rb

@@ -0,0 +1,62 @@
+require 'monitor'
+
+require 'active_support/concern'
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+require 'active_model'
+
+require 'zermelo/associations/class_methods'
+
+require 'zermelo/records/instance_methods'
+require 'zermelo/records/class_methods'
+require 'zermelo/records/type_validator'
+
+# TODO escape ids and index_keys -- shouldn't allow bare :
+
+# TODO callbacks on before/after add/delete on association?
+
+# TODO optional sort via Redis SORT, first/last for has_many via those
+
+# TODO get DIFF working for exclusion case against ZSETs
+
+module Zermelo
+
+  module Records
+
+    module Base
+
+      extend ActiveSupport::Concern
+
+      include Zermelo::Records::InstMethods
+
+      included do
+        include ActiveModel::AttributeMethods
+        extend ActiveModel::Callbacks
+        include ActiveModel::Dirty
+        include ActiveModel::Validations
+        include ActiveModel::Validations::Callbacks
+
+        # include ActiveModel::MassAssignmentSecurity
+
+        @lock = Monitor.new
+
+        extend Zermelo::Records::ClassMethods
+        extend Zermelo::Associations::ClassMethods
+
+        attr_accessor :attributes
+
+        define_model_callbacks :create, :update, :destroy
+
+        attribute_method_suffix  "="  # attr_writers
+        # attribute_method_suffix  ""   # attr_readers # DEPRECATED
+
+        validates_with Zermelo::Records::TypeValidator
+
+        define_attributes :id => :string
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/records/class_methods.rb

@@ -0,0 +1,127 @@
+require 'forwardable'
+require 'securerandom'
+
+require 'zermelo'
+
+require 'zermelo/backends/influxdb_backend'
+require 'zermelo/backends/redis_backend'
+
+require 'zermelo/records/key'
+
+module Zermelo
+
+  module Records
+
+    module ClassMethods
+
+      extend Forwardable
+
+      def_delegators :filter, :intersect, :union, :diff, :sort,
+                       :find_by_id, :find_by_ids, :find_by_id!, :find_by_ids!,
+                       :page, :all, :each, :collect, :map,
+                       :select, :find_all, :reject, :destroy_all,
+                       :ids, :count, :empty?, :exists?,
+                       :associated_ids_for
+
+      def generate_id
+        return SecureRandom.uuid if SecureRandom.respond_to?(:uuid)
+        # from 1.9 stdlib
+        ary = SecureRandom.random_bytes(16).unpack("NnnnnN")
+        ary[2] = (ary[2] & 0x0fff) | 0x4000
+        ary[3] = (ary[3] & 0x3fff) | 0x8000
+        "%08x-%04x-%04x-%04x-%04x%08x" % ary
+      end
+
+      def add_id(id)
+        backend.add(ids_key, id.to_s)
+      end
+
+      def delete_id(id)
+        backend.delete(ids_key, id.to_s)
+      end
+
+      def attribute_types
+        ret = nil
+        @lock.synchronize do
+          ret = (@attribute_types ||= {}).dup
+        end
+        ret
+      end
+
+      def lock(*klasses, &block)
+        klasses += [self] unless klasses.include?(self)
+        backend.lock(*klasses, &block)
+      end
+
+      def transaction(&block)
+        failed = false
+
+        backend.begin_transaction
+
+        begin
+          yield
+        rescue Exception => e
+          backend.abort_transaction
+          p e.message
+          puts e.backtrace.join("\n")
+          failed = true
+        ensure
+          backend.commit_transaction unless failed
+        end
+
+        # TODO include exception info
+        raise "Transaction failed" if failed
+      end
+
+      def backend
+        raise "No data storage backend set for #{self.name}" if @backend.nil?
+        @backend
+      end
+
+      protected
+
+      def define_attributes(options = {})
+        options.each_pair do |key, value|
+          raise "Unknown attribute type ':#{value}' for ':#{key}'" unless
+            Zermelo.valid_type?(value)
+          self.define_attribute_methods([key])
+        end
+        @lock.synchronize do
+          (@attribute_types ||= {}).update(options)
+        end
+      end
+
+      def set_backend(backend_type)
+        @backend ||= case backend_type.to_sym
+        when :influxdb
+          Zermelo::Backends::InfluxDBBackend.new
+        when :redis
+          Zermelo::Backends::RedisBackend.new
+        end
+      end
+
+      private
+
+      def ids_key
+        @ids_key ||= Zermelo::Records::Key.new(:klass => class_key, :name => 'ids',
+          :type => :set, :object => :attribute)
+      end
+
+      def class_key
+        self.name.demodulize.underscore
+      end
+
+      def load(id)
+        object = self.new
+        object.load(id) ? object : nil
+      end
+
+      def filter
+        backend.filter(ids_key, self)
+      end
+
+    end
+
+  end
+
+end

data/lib/zermelo/records/collection.rb

@@ -0,0 +1,14 @@
+module Zermelo
+  module Records
+    # high-level abstraction for a set or list of record ids
+    class Collection
+      attr_reader :klass, :name, :type
+
+      def initialize(opts = {})
+        @klass  = opts[:class]
+        @name   = opts[:name]
+        @type   = opts[:type]
+      end
+    end
+  end
+end

data/lib/zermelo/records/errors.rb

@@ -0,0 +1,24 @@
+module Zermelo
+  module Records
+    module Errors
+
+      class RecordNotFound < RuntimeError
+        attr_reader :klass, :id
+
+        def initialize(k, i)
+          @klass = k
+          @id = i
+        end
+      end
+
+      class RecordsNotFound < RuntimeError
+        attr_reader :klass, :ids
+
+        def initialize(k, i)
+          @klass = k
+          @ids = i
+        end
+      end
+   end
+ end
+end

data/lib/zermelo/records/influxdb_record.rb

@@ -0,0 +1,35 @@
+require 'active_support/concern'
+
+require 'zermelo/records/base'
+
+# a record is a row in a time series (named for the record class)
+
+# all attributes are stored as fields in that row
+
+# a save will delete (if required) and create the row
+
+# if time field does not exist, this will be created automatically by influxdb
+
+# indexing -- not really relevant until query building has been worked on, but
+# everything in the influxdb query language should be supportable, maybe those
+# just indicate what should be queryable?
+
+# TODO: ensure time_precision is set for the incoming data
+
+# class level values are in other time series (with similar names to the
+# related redis sets)
+
+module Zermelo
+  module Records
+    module InfluxDBRecord
+      extend ActiveSupport::Concern
+
+      include Zermelo::Records::Base
+
+      included do
+        set_backend :influxdb
+      end
+
+    end
+  end
+end