hyperion-riak 0.1.0

data/lib/hyperion/riak.rb

@@ -0,0 +1,9 @@
+require 'hyperion/riak/datastore'
+
+module Hyperion
+  module Riak
+    def self.new(opts={})
+      Datastore.new(opts)
+    end
+  end
+end

data/lib/hyperion/riak/datastore.rb

@@ -0,0 +1,138 @@
+require 'hyperion'
+require 'hyperion/key'
+require 'hyperion/riak/map_reduce_js'
+require 'riak'
+require 'hyperion/riak/optimized_filter_order'
+
+module Hyperion
+  module Riak
+    class Datastore
+      def initialize(opts={})
+        opts ||= {}
+        @app = opts[:app] || ''
+        @client = ::Riak::Client.new(opts.reject {|k, v| k == :app})
+        @buckets = {}
+      end
+
+      def save(records)
+        records.map do |record|
+          Hyperion.new?(record) ? create(record) : update(record)
+        end
+      end
+
+      def find_by_key(key)
+        kind, riak_key = Hyperion::Key.decompose_key(key)
+        robject = bucket(kind).get(riak_key)
+        record_from_db(kind, robject.key, robject.data)
+      end
+
+      def find(query)
+        mr = new_mapreduce_with_returned_result(query)
+        mr.run.map do |record|
+          record_from_db(query.kind, record.delete('riak_key'), record)
+        end
+      end
+
+      def delete_by_key(key)
+        kind, riak_key = Hyperion::Key.decompose_key(key)
+        delete_with_riak_key(kind, riak_key)
+        nil
+      end
+
+      def delete(query)
+        mr = new_mapreduce_with_returned_result(query)
+        mr.run.each do |record|
+          delete_with_riak_key(query.kind, record['riak_key'])
+        end
+        nil
+      end
+
+      def count(query)
+        mr = new_mapreduce(query)
+        mr.reduce(MapReduceJs.count, :keep => true)
+        mr.run.first
+      end
+
+      private
+
+      def create(record)
+        kind = record[:kind]
+        robject = bucket(kind).new
+        store(kind, robject, record_to_db(record))
+      end
+
+      def update(record)
+        kind, riak_key = Hyperion::Key.decompose_key(record[:key])
+        robject = bucket(kind).get(riak_key)
+        store(kind, robject, robject.data.merge(record_to_db(record)))
+      end
+
+      def store(kind, robject, record_data)
+        robject.data = record_data
+        robject.indexes = record_data_to_index(record_data)
+        robject.store
+        record_from_db(kind, robject.key, robject.data)
+      end
+
+      def record_data_to_index(data)
+        data.reduce({}) do |new_record, (key, value)|
+          new_record[index_name(key)] = value
+          new_record
+        end
+      end
+
+      def delete_with_riak_key(kind, key)
+        bucket(kind).delete(key)
+      end
+
+      def new_mapreduce(query)
+        mr = ::Riak::MapReduce.new(@client)
+        add_query_filters(mr, query)
+        sorts = query.sorts
+        mr.reduce(MapReduceJs.sort(sorts)) unless sorts.empty?
+        mr.reduce(MapReduceJs.offset(query.offset)) if query.offset
+        mr.reduce(MapReduceJs.limit(query.limit)) if query.limit
+        mr
+      end
+
+      def add_query_filters(mr, query)
+        bucket_name = bucket_name(query.kind)
+        optimizer = OptimizedFilterOrder.new(query.filters, bucket_name)
+        field_index = index_name(optimizer.optimal_index_field)
+        mr.index(bucket_name, field_index, optimizer.optimal_index_value)
+        mr.map(MapReduceJs.filter(optimizer.filters))
+      end
+
+      def index_name(field_name)
+        if field_name != '$bucket'
+          "#{field_name}_bin"
+        else
+          field_name
+        end
+      end
+
+      def new_mapreduce_with_returned_result(query)
+        mr = new_mapreduce(query)
+        mr.reduce(MapReduceJs.pass_thru, :keep => true)
+      end
+
+      def record_to_db(record)
+        record.reject {|k, v| [:kind, :key].include?(k)}
+      end
+
+      def record_from_db(kind, riak_key, data)
+        key = Hyperion::Key.compose_key(kind, riak_key)
+        data.merge(:kind => kind, :key => key)
+      end
+
+      def bucket(kind)
+        name = bucket_name(kind)
+        @buckets[name] ||= @client.bucket(name)
+      end
+
+      def bucket_name(kind)
+        @app.to_s + kind
+      end
+    end
+  end
+end

data/lib/hyperion/riak/map_reduce_js.rb

@@ -0,0 +1,53 @@
+require 'erb'
+
+module Hyperion
+  module Riak
+    class MapReduceJs
+      class << self
+
+        def filter(filters)
+          template(:filter).result(binding)
+        end
+
+        def sort(sorts)
+          template(:sort).result(binding)
+        end
+
+        def offset(offset)
+          template(:offset).result(binding)
+        end
+
+        def offset(offset)
+          template(:offset).result(binding)
+        end
+
+        def limit(limit)
+          template(:limit).result(binding)
+        end
+
+        def count
+          template(:count).result
+        end
+
+        def pass_thru
+          template(:pass_thru).result
+        end
+
+        private
+
+        def template(name)
+          @templates ||= {}
+          @templates[name] ||= ERB.new(file_contents(template_path(name)))
+        end
+
+        def template_path(name)
+          File.expand_path(File.join('..', 'map_reduce', "#{name}.js.erb"), __FILE__)
+        end
+
+        def file_contents(path)
+          File.open(path, 'rb') { |f| f.read }
+        end
+      end
+    end
+  end
+end

data/lib/hyperion/riak/optimized_filter_order.rb

@@ -0,0 +1,111 @@
+require 'hyperion/riak/optimized_range_filters'
+
+module Hyperion
+  module Riak
+    class OptimizedFilterOrder
+
+      def initialize(filters, bucket_name)
+        @bucket_name = bucket_name
+        @filters = filters
+      end
+
+      def optimal_strategy
+        @optimal_strategy ||= OPTIMAL_ORDER.map do |strategy_klass|
+          strategy_klass.new(@filters, @bucket_name)
+        end.find do |strategy|
+          strategy.can_optimize?
+        end
+      end
+
+      def optimal_index_field
+        optimal_strategy.optimal_index_field
+      end
+
+      def optimal_index_value
+        optimal_strategy.optimal_index_value
+      end
+
+      def filters
+        optimal_strategy.filters
+      end
+
+      private
+    end
+
+    class EqualsStrategy
+      def initialize(filters, bucket_name)
+        @filters = filters
+      end
+
+      def can_optimize?
+        !first_equals_filter.nil?
+      end
+
+      def optimal_index_field
+        first_equals_filter.field
+      end
+
+      def optimal_index_value
+        first_equals_filter.value.to_s
+      end
+
+      def filters
+        @remaining_filters ||= (@filters - [first_equals_filter])
+      end
+
+      private
+
+      def first_equals_filter
+        @first_equals_filter ||= @filters.find { |f| f.operator == "=" }
+      end
+    end
+
+    class RangeStrategy
+      def initialize(filters, bucket_name)
+        @filters = filters
+        @optimizer = OptimizedRangeFilters.new(filters)
+      end
+
+      def can_optimize?
+        @optimizer.less_than_filter && @optimizer.greater_than_filter
+      end
+
+      def optimal_index_field
+        @optimizer.less_than_filter.field
+      end
+
+      def optimal_index_value
+        @value ||= @optimizer.less_than_filter.value.to_s .. @optimizer.greater_than_filter.value.to_s
+      end
+
+      def filters
+        @optimizer.remaining_filters
+      end
+    end
+
+    class BucketStrategy
+      def initialize(filters, bucket_name)
+        @filters = filters
+        @bucket_name = bucket_name
+      end
+
+      def can_optimize?
+        true
+      end
+
+      def optimal_index_field
+        '$bucket'
+      end
+
+      def optimal_index_value
+        @bucket_name
+      end
+
+      def filters
+        @filters
+      end
+    end
+
+    OPTIMAL_ORDER = [EqualsStrategy, RangeStrategy, BucketStrategy]
+  end
+end

data/lib/hyperion/riak/optimized_range_filters.rb

@@ -0,0 +1,42 @@
+module Hyperion
+  module Riak
+    class OptimizedRangeFilters
+      def initialize(filters)
+        @filters = filters
+      end
+
+      def remaining_filters
+        @remaining_filters ||= @filters - [less_than_filter, greater_than_filter]
+      end
+
+      def less_than_filter
+        @less_than_filter ||= find_first_match(less_than_candidates, greater_than_candidates)
+      end
+
+      def greater_than_filter
+        @greater_than_filter ||= find_first_match(greater_than_candidates, less_than_candidates)
+      end
+
+      private
+
+      def find_first_match(filters_to_search, filters_to_match_against)
+        filters_to_search.find do |f1|
+          filters_to_match_against.any? { |f2| f1.field == f2.field }
+        end
+      end
+
+      def greater_than_candidates
+        @greater_than_filters ||= @filters.select do |filter|
+          filter.operator == '>'
+        end
+      end
+
+      def less_than_candidates
+        @less_than_candidates ||= @filters.select do |filter|
+          filter.operator == '<'
+        end
+      end
+
+    end
+  end
+end

data/lib/hyperion/riak/spec_helper.rb

@@ -0,0 +1,26 @@
+def test_app_name
+  '_HTEST_'
+end
+
+BUCKETS = ['testing', 'other_testing']
+
+def empty_buckets(ds)
+  client = ds.instance_variable_get(:@client)
+  BUCKETS.each do |bucket_name|
+    bucket_name = ds.send(:bucket_name, bucket_name)
+    bucket = client.bucket(bucket_name)
+    bucket.get_index('$bucket', bucket_name).each do |key|
+      bucket.delete(key)
+    end
+  end
+end
+
+def with_testable_riak_datastore
+  ds = Hyperion.new_datastore(:riak, :app => test_app_name, :protocol => :pbc)
+  around :each do |example|
+    Hyperion.datastore = ds
+    example.run
+    empty_buckets(ds)
+    Hyperion.datastore = nil
+  end
+end

data/spec/hyperion/riak/optimized_filter_order_spec.rb

@@ -0,0 +1,122 @@
+require 'hyperion/filter'
+require 'hyperion/riak/optimized_filter_order'
+
+describe Hyperion::Riak::OptimizedFilterOrder do
+
+  def filter(field, operator, value)
+    Hyperion::Filter.new(field, operator, value)
+  end
+
+  context "no filters" do
+    it 'returns "$bucket" for optimal_index_field' do
+      filters = []
+      bucket_name = "hamburgers"
+
+      o = described_class.new(filters, bucket_name)
+      o.optimal_index_field.should == '$bucket'
+    end
+
+    it 'returns bucket name for optimal_index_value' do
+      filters = []
+      bucket_name = "hamburgers"
+
+      o = described_class.new(filters, bucket_name)
+      o.optimal_index_value.should == bucket_name
+    end
+
+    it 'returns an empty collection for filters' do
+      filters = []
+      bucket_name = "hamburgers"
+
+      o = described_class.new(filters, bucket_name)
+      o.filters.should == []
+    end
+  end
+
+  context 'optimizes for an equals filter' do
+    it 'returns the index field' do
+      filters = [filter(:int, '=', 1)]
+      o = described_class.new(filters, '')
+      o.optimal_index_field.should == :int
+    end
+
+    it 'returns bucket name for optimal_index_value' do
+      filters = [filter(:int, '=', 1)]
+      bucket_name = "hamburgers"
+
+      o = described_class.new(filters, bucket_name)
+      o.optimal_index_value.should == '1'
+    end
+
+     it 'excludes the optimized filter' do
+      filters = [filter(:int, '=', 1)]
+      o = described_class.new(filters, '')
+      o.filters.should be_empty
+     end
+  end
+
+  context 'optimizes for a range filter' do
+    it 'returns the index field' do
+      filters = [filter(:int, '<', 1), filter(:int, '>', 2), filter(:int, '?', 3)]
+      o = described_class.new(filters, '')
+      o.optimal_index_field.should == :int
+    end
+
+    it 'returns bucket name for optimal_index_value' do
+      filters = [filter(:int, '<', 1), filter(:int, '>', 2), filter(:int, '?', 3)]
+
+      o = described_class.new(filters, '')
+      o.optimal_index_value.should == ('1'..'2')
+    end
+
+     it 'excludes the optimized filters' do
+      leftover_filter = filter(:int, '?', 3)
+      filters = [filter(:int, '<', 1), filter(:int, '>', 2), leftover_filter]
+      o = described_class.new(filters, '')
+      o.filters.should == [leftover_filter]
+     end
+  end
+
+  context 'cannot be optimized' do
+    it 'returns "$bucket" for index_field if no optimal filter' do
+      filters = [filter(:int, '?', 1)]
+      o = described_class.new(filters, '')
+      o.optimal_index_field.should == '$bucket'
+    end
+
+    it 'returns bucket name for optimal_index_value' do
+      filters = [filter(:int, '?', 1)]
+      o = described_class.new(filters, 'cheeseburgers')
+      o.optimal_index_value.should == 'cheeseburgers'
+    end
+
+     it 'filters contains all filters' do
+      filters = [filter(:int, '?', 1)]
+      o = described_class.new(filters, '')
+      o.filters.should == filters
+     end
+  end
+
+  it 'chooses equals filter over bucket' do
+    equal_filter = filter(:data, '=', 3)
+    o = described_class.new([equal_filter], '')
+    o.optimal_index_field.should == :data
+    o.optimal_index_value.should == '3'
+    o.filters.should == []
+  end
+
+  it 'chooses equals filter over range filter' do
+    equal_filter = filter(:data, '=', 3)
+    filters = [filter(:int, '<', 1), filter(:int, '>', 2)]
+    o = described_class.new(filters + [equal_filter], '')
+    o.optimal_index_field.should == :data
+    o.optimal_index_value.should == '3'
+    o.filters.should == filters
+  end
+
+  it 'chooses range filter over bucket filter' do
+    filters = [filter(:int, '<', 1), filter(:int, '>', 2)]
+    o = described_class.new(filters, '')
+    o.optimal_index_field.should == :int
+  end
+end

data/spec/hyperion/riak/optimized_range_filters_spec.rb

@@ -0,0 +1,87 @@
+require 'hyperion/filter'
+require 'hyperion/riak/optimized_range_filters'
+
+describe Hyperion::Riak::OptimizedRangeFilters do
+  let(:test_less_than_filter) { filter(:test, '<', 1) }
+  let(:test_greater_than_filter) { filter(:test, '>', 1) }
+  let(:test_equals_filter) { filter(:test, '=', 1) }
+  let(:other_test_less_than_filter) { filter(:other_test, '<', 1) }
+  let(:other_test_greater_than_filter) { filter(:other_test, '>', 1) }
+
+  it 'returns the filters' do
+    optimizer = described_class.new([])
+    optimizer.remaining_filters.should == []
+  end
+
+  def filter(field, operator, value)
+    Hyperion::Filter.new(field, operator, value)
+  end
+
+  it 'returns the optimal filters' do
+    optimizer = described_class.new([test_less_than_filter, test_greater_than_filter])
+    optimizer.less_than_filter.should == test_less_than_filter
+    optimizer.greater_than_filter.should == test_greater_than_filter
+    optimizer.remaining_filters.should == []
+  end
+
+  it 'returns the optimal filters' do
+    optimizer = described_class.new([test_greater_than_filter, test_less_than_filter])
+    optimizer.less_than_filter.should == test_less_than_filter
+    optimizer.greater_than_filter.should == test_greater_than_filter
+    optimizer.remaining_filters.should == []
+  end
+
+  it 'returns non-range filter as remaining filter' do
+    optimizer = described_class.new([
+      test_greater_than_filter,
+      test_less_than_filter,
+      test_equals_filter
+    ])
+    optimizer.remaining_filters.should == [test_equals_filter]
+  end
+
+  it 'does not return a less than filter if there is no greater than' do
+    filters = [test_less_than_filter, test_equals_filter]
+    optimizer = described_class.new(filters)
+    optimizer.remaining_filters.should == filters
+    optimizer.less_than_filter.should be_nil
+  end
+
+  it 'does not return a less than filter if there is no matching greater than' do
+    filters = [
+      filter(:test, '<', 1),
+      filter(:other_test, '>', 1),
+      test_equals_filter
+    ]
+    optimizer = described_class.new(filters)
+    optimizer.remaining_filters.should == filters
+    optimizer.less_than_filter.should be_nil
+    optimizer.greater_than_filter.should be_nil
+  end
+
+  it 'returns a matching range when there are other candidates' do
+    filters = [
+      test_less_than_filter,
+      other_test_less_than_filter,
+      other_test_greater_than_filter,
+      test_equals_filter
+    ]
+    optimizer = described_class.new(filters)
+    optimizer.remaining_filters.should == [test_less_than_filter, test_equals_filter]
+    optimizer.less_than_filter.should == other_test_less_than_filter
+    optimizer.greater_than_filter.should == other_test_greater_than_filter
+  end
+
+  it 'returns a matching range when there are other candidates' do
+    filters = [
+      test_greater_than_filter,
+      other_test_greater_than_filter,
+      test_less_than_filter,
+      test_equals_filter
+    ]
+    optimizer = described_class.new(filters)
+    optimizer.remaining_filters.should == [other_test_greater_than_filter, test_equals_filter]
+    optimizer.less_than_filter.should == test_less_than_filter
+    optimizer.greater_than_filter.should == test_greater_than_filter
+  end
+end

data/spec/hyperion/riak_spec.rb

@@ -0,0 +1,12 @@
+require 'hyperion/dev/ds_spec'
+require 'hyperion/riak'
+require 'hyperion/riak/spec_helper'
+
+describe Hyperion::Riak do
+
+  context 'live' do
+    with_testable_riak_datastore
+
+    include_examples 'Datastore'
+  end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: hyperion-riak
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Myles Megyesi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+- !ruby/object:Gem::Dependency
+  name: hyperion-api
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: riak-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+description: Riak datastore for Hyperion
+email:
+- myles@8thlight.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/hyperion/riak.rb
+- lib/hyperion/riak/map_reduce_js.rb
+- lib/hyperion/riak/optimized_filter_order.rb
+- lib/hyperion/riak/optimized_range_filters.rb
+- lib/hyperion/riak/spec_helper.rb
+- lib/hyperion/riak/datastore.rb
+- spec/hyperion/riak_spec.rb
+- spec/hyperion/riak/optimized_filter_order_spec.rb
+- spec/hyperion/riak/optimized_range_filters_spec.rb
+homepage: https://github.com/mylesmegyesi/hyperion-ruby
+licenses:
+- Eclipse Public License
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Riak datastore for Hypeiron
+test_files:
+- spec/hyperion/riak_spec.rb
+- spec/hyperion/riak/optimized_filter_order_spec.rb
+- spec/hyperion/riak/optimized_range_filters_spec.rb