dm-adapter-simpledb 1.3.0 → 1.4.0

data/lib/dm-adapter-simpledb/rake.rb

@@ -1,43 +1,2 @@
 namespace :simpledb do
-  desc "Migrate records to be compatable with current DM/SimpleDB adapter"
-  task :migrate, :domain do |t, args|
-    raise "THIS IS A WORK IN PROGRESS AND WILL DESTROY YOUR DATA"
-    require 'progressbar'
-    require 'right_aws'
-    require 'dm-adapter-simpledb/record'
-
-    puts "Initializing connection..."
-    domain = args.domain
-    sdb = RightAws::SdbInterface.new
-    puts "Counting records..."
-    num_legacy_records = 0
-    query = "select count(*) from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
-    next_token = nil
-    while(results = sdb.select(query, next_token)) do
-      next_token = results[:next_token]
-      count = results[:items].first["Domain"]["Count"].first.to_i
-      num_legacy_records += count
-      break if next_token.nil?
-    end
-    puts "Found #{num_legacy_records} to migrate"
-
-    pbar = ProgressBar.new("migrate", num_legacy_records)
-    query = "select * from #{domain} where (simpledb_type is not null) and (__dm_metadata is null)"
-    while(results = sdb.select(query, next_token)) do
-      next_token = results[:next_token]
-      items = results[:items]
-      items.each do |item|
-        legacy_record = DmAdapterSimpledb::Record.from_simpledb_hash(item)
-        new_record    = legacy_record.migrate
-        updates       = new_record.writable_attributes
-        deletes       = new_record.deletable_attributes
-        sdb.put_attributes(domain, new_record.item_name, updates)
-        sdb.delete_attributes(domain, new_record.item_name, deletes)
-        pbar.inc
-      end
-      break if next_token.nil?
-    end
-    pbar.finish
-
-  end
 end

data/lib/dm-adapter-simpledb/record.rb

@@ -315,7 +315,7 @@ module DmAdapterSimpledb
     private
 
     def replace_newline_placeholders(value)
-      value.gsub("[[[NEWLINE]]]", "\n")
+      value && value.gsub("[[[NEWLINE]]]", "\n")
     end
   end
 

data/lib/dm-adapter-simpledb/table.rb

@@ -20,8 +20,9 @@ module DmAdapterSimpledb
 
     # Returns a string so we know what type of
     def simpledb_type
-      model.storage_name(repository_name)
+      model.storage_name(DataMapper.repository.name)
     end
+    alias_method :storage_name, :simpledb_type
 
     def repository_name
       # TODO this should probably take into account the adapter

data/lib/dm-adapter-simpledb/utils.rb

@@ -1,5 +1,49 @@
 module DmAdapterSimpledb
   module Utils
+    class NullObject
+      def method_missing(*args, &block)
+        self
+      end
+    end
+
+    class NullSdbInterface
+      def initialize(logger=NullObject.new)
+        @logger = logger
+      end
+
+      def select(*args, &block)
+        @logger.debug "[SELECT] #{args.inspect}"
+        {
+          :items => []
+        }
+      end
+
+      def get_attributes(*args, &block)
+        @logger.debug "[GET_ATTRIBUTES] #{args.inspect}"
+        {}
+      end
+
+      def list_domains(*args, &block)
+        @logger.debug "[LIST_DOMAINS] #{args.inspect}"
+        {}
+      end
+
+      def put_attributes(*args, &block)
+        @logger.debug "[PUT_ATTRIBUTES] #{args.inspect}"
+        {}
+      end
+
+      def delete_attributes(*args, &block)
+        @logger.debug "[DELETE_ATTRIBUTES] #{args.inspect}"
+        {}
+      end
+
+      def create_domain(*args, &block)
+        @logger.debug "[CREATE_DOMAIN] #{args.inspect}"
+        {}
+      end
+    end
+
     def transform_hash(original, options={}, &block)
       original.inject({}){|result, (key,value)|
         value = if (options[:deep] && Hash === value) 

data/lib/dm-adapter-simpledb/where_expression.rb

@@ -0,0 +1,180 @@
+module DmAdapterSimpledb
+  class WhereExpression
+    include DataMapper::Query::Conditions
+
+    attr_reader   :conditions
+    attr_accessor :logger
+
+    def initialize(conditions, options={})
+      @conditions = conditions
+      @logger     = options.fetch(:logger){ DataMapper.logger }
+    end
+
+    def to_s
+      node_to_expression(conditions)
+    end
+
+    def unsupported_conditions(node=conditions, top=true)
+      case node
+      when InclusionComparison
+        if node.value.is_a?(Range) && node.value.exclude_end?
+          logger.warn "Exclusive ranges are not supported natively by the SimpleDB adapter"
+          node.dup
+        end
+      when RegexpComparison
+        logger.warn "Regexp comparisons are not supported natively by the SimpleDB adapter"
+        node.dup
+      when AbstractOperation
+        op_copy = node.dup
+        op_copy.clear
+        operands_copy = 
+          node.operands.map{|o| unsupported_conditions(o,false)}.compact
+        if operands_copy.size > 0
+          op_copy.operands.merge(operands_copy)
+          op_copy
+        else
+          top ? Operation.new(:and) : nil
+        end
+      else top ? Operation.new(:and) : nil
+      end
+    end
+
+    private
+
+    def node_to_expression(node)
+      case node
+      when AbstractOperation  then operation_to_expression(node)
+      when AbstractComparison then comparison_to_expression(node)
+      when Array then array_to_expression(node)
+      when nil then nil
+      else raise NotImplementedError, "Unrecognized node: #{node.inspect}"
+      end
+    end
+
+    def operation_to_expression(operation)
+      case operation
+      when NotOperation
+        operand = operation.sorted_operands.first # NOT only allowed to have 1 operand
+        if operand.is_a?(EqualToComparison)
+          if operand.value.nil?
+            comparison_to_expression(operand, "IS NOT")
+          else
+            comparison_to_expression(operand, "!=")
+          end
+        elsif empty_inclusion?(operand)
+          nil
+        else
+          "NOT #{node_to_expression(operand)}"
+        end
+      when AndOperation then
+        join_operands(operation, "AND")
+      when OrOperation then
+        join_operands(operation, "OR")
+      when nil then nil
+      else raise NotImplementedError, "Unhandled operation: #{operation.inspect}"
+      end
+    end
+
+    def join_operands(operation, delimiter)
+      operands = operation.sorted_operands
+      joined_operands = Array(operands).map{|o| 
+        catch(:unsupported) { node_to_expression(o) }
+      }.compact.join(" #{delimiter} ")
+      if operation.parent
+        "( #{joined_operands} )"
+      else
+        joined_operands
+      end
+    end
+
+    def comparison_to_expression(
+        comparison, 
+        operator=comparison_operator(comparison),
+        values  = value_to_expression(comparison.value))
+      field = SDBTools::Selection.quote_name(comparison.subject.field)
+      if empty_inclusion?(comparison)
+        "#{field} IS NULL"
+      else
+        "#{field} #{operator} #{values}"
+      end
+    end
+
+    def array_to_expression(array)
+      template, *replacements = *array.dup
+      if replacements.size == 1 && replacements.first.is_a?(Array)
+        replacements = replacements.first
+      end
+      if replacements.size == 1 && replacements[0].is_a?(Hash)
+        fill_template_from_hash(template, replacements[0])
+      else
+        fill_template_from_array(template, replacements)
+      end
+    end
+
+    def fill_template_from_array(template, replacements)
+      template.to_s.gsub("?") {|match| quote_value(replacements.shift.to_s) }
+    end
+
+    def fill_template_from_hash(template, replacements)
+      template.to_s.gsub(/:\w+/) { |match|
+        quote_value(replacements.fetch(match[1..-1].to_sym){match})
+      }
+    end
+
+    def value_to_expression(value)
+      case value
+      when nil then "NULL"
+      when Range then
+        "#{quote_value(value.begin)} AND #{quote_value(value.end)}"
+      when Array then
+        value = value.map{|v| value_to_expression(v) }.join(", ")
+        value = "(#{value})"
+      else quote_value(value)
+      end
+    end
+
+    def quote_value(value)
+      SDBTools::Selection.quote_value(value)
+    end
+
+    def comparison_operator(comparison)
+      case comparison
+      when EqualToComparison              then
+        case comparison.value
+        when nil then "IS"
+        else "="
+        end
+      when GreaterThanComparison          then ">"
+      when LessThanComparison             then "<"
+      when GreaterThanOrEqualToComparison then ">="
+      when LessThanOrEqualToComparison    then "<="
+      when LikeComparison                 then "LIKE"
+      when InclusionComparison            then
+        case comparison.value
+        when Range then 
+          if comparison.value.exclude_end?
+            # We have tried to support exclusive ranges by simply adding
+            # the range to the unsupported_conditions list so that excluded
+            # values will be caught in post-filtering.  However, DataMapper
+            # is apparently casting the range begin/end values to Strings, even
+            # when the property type is an Integer. 
+
+            # Or it may be that something else is going wrong.  But the upshot
+            # is that the post-filtering step doesn't work.
+            raise NotImplementedError, 
+                  "Exclusive ranges are not supported by the SimpleDB adapter"
+          else
+            "BETWEEN"
+          end
+        else "IN"
+        end
+      when RegexpComparison then throw :unsupported
+      else raise NotImplementedError, "Unhandled comparison: #{comparison.inspect}"
+      end
+    end
+
+    def empty_inclusion?(node)
+      node.is_a?(InclusionComparison) && Array(node.value).empty?
+    end
+  end
+end

data/scripts/console

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require File.expand_path('../lib/dm-adapter-simpledb', File.dirname(__FILE__))
+require 'logger'
+require 'irb'
+
+include DataMapper
+
+class Post
+  include Resource
+  
+  property :title, String, :key => true
+  property :body,  Text
+end
+
+DataMapper.setup(
+  :default,
+  :domain  => "example",
+  :adapter => 'simpledb',
+  :null    => true,
+  :logger  => ::Logger.new($stderr, ::Logger::DEBUG))
+
+IRB.start

data/scripts/limits_benchmark

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require File.expand_path('../lib/dm-adapter-simpledb', File.dirname(__FILE__))
+require 'logger'
+
+include DataMapper
+
+class Post
+  include Resource
+  
+  property :title, String, :key => true
+  property :body,  Text
+end
+
+logger = ::Logger.new($stderr, ::Logger::DEBUG)
+
+DataMapper.setup(
+  :default,
+  :domain        => "dm_simpledb_adapter_benchmark",
+  :adapter       => 'simpledb',
+  :create_domain => true,
+  :logger        => logger,
+  :batch_limit   => 10)
+
+logger.info "Writing records"
+100.times do |i|
+  Post.create(:title => "title#{i}", :body => "body#{i}")
+end
+
+logger.info "Waiting a bit for consistency"
+sleep 2
+
+SDBTools::Transaction.on_close = 
+  SDBTools::Transaction.log_transaction_close(logger)
+SDBTools::Transaction.open("benchmark limited query") do |t|
+  # With a batch limit of 10, this should result in two queries of LIMIT 10 and
+  # one with LIMIT 5
+  query      = Post.all(:limit => 25)
+  item_count = query.to_a.size
+  logger.info "Found #{item_count} items"
+end
+
+SDBTools::Transaction.on_close = 
+  SDBTools::Transaction.log_transaction_close(logger)
+SDBTools::Transaction.open("benchmark query with limit and offset") do |t|
+  # With a batch limit of 10, this should result in two queries of LIMIT 10 and
+  # one with LIMIT 5
+  query      = Post.all(:limit => 25, :order => :title.asc, :offset => 25)
+  item_count = query.to_a.size
+  logger.info "Found #{item_count} items"
+end

data/scripts/union_benchmark

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require File.expand_path('../lib/dm-adapter-simpledb', File.dirname(__FILE__))
+require 'logger'
+
+include DataMapper
+
+class Post
+  include Resource
+  
+  property :title, String, :key => true
+  property :body,  Text
+end
+
+logger = ::Logger.new($stderr, ::Logger::DEBUG)
+
+DataMapper.setup(
+  :default,
+  :domain        => "dm_simpledb_adapter_benchmark",
+  :adapter       => 'simpledb',
+  :create_domain => true,
+  :logger        => logger)
+
+logger.info "Writing records"
+100.times do |i|
+  Post.create(:title => "title#{i}", :body => "body#{i}")
+end
+
+logger.info "Waiting a bit for consistency"
+sleep 2
+
+logger.info "benchmarking OR"
+SDBTools::Transaction.on_close = 
+  SDBTools::Transaction.log_transaction_close(logger)
+SDBTools::Transaction.open("benchmark OR") do |t|
+  query      = Post.all(:title.like => "%2%") | Post.all(:title.like => "%5%")
+  item_count = query.to_a.size
+  logger.info "Found #{item_count} items"
+end

data/spec/integration/compliance_spec.rb

@@ -8,6 +8,7 @@ describe DataMapper::Adapters::SimpleDBAdapter do
     @adapter = DataMapper::Repository.adapters[:default]
     @old_consistency_policy = @adapter.consistency_policy
     @adapter.consistency_policy = :automatic
+    # @adapter.logger = ::Logger.new($stderr)
   end
 
   after :all do

data/spec/integration/limit_and_order_spec.rb

@@ -40,7 +40,7 @@ describe 'with multiple records saved' do
 
   it 'should handle max item if limit is large case' do
     persons = Hero.all(:limit => 150)
-    persons.length.should ==3
+    persons.length.should == 3
   end
 
   it 'should handle ordering asc results with a limit' do

data/spec/integration/simpledb_adapter_spec.rb

@@ -138,33 +138,15 @@ EOF
       persons.length.should == 0
     end
 
-    describe '#query' do
-      before(:each) do
-        @domain = Friend.repository(:default).adapter.sdb_options[:domain]
-      end
-      it "should return an array of records" do
-        records = Friend.repository(:default).adapter.query("SELECT age, wealth from #{@domain} where age = '25'")
-        records.should == [{"wealth"=>["25.0"], "age"=>["25"]}]
-      end
-      it "should return empty array if no matches" do
-        records = Friend.repository(:default).adapter.query("SELECT age, wealth from #{@domain} where age = '15'")
-        records.should be_empty
-      end
-      it "should raise an error if query is invalid" do
-        lambda do
-          records = Friend.repository(:default).adapter.query("SELECT gaga")
-        end.should raise_error(RightAws::AwsError)
-      end
-    end
     describe 'aggregate' do
       it "should respond to count(*)" do
         Friend.count.should == 1
       end
       it "should not respond to any other aggregates" do
-        lambda { Friend.min(:age) }.should raise_error(ArgumentError)
-        lambda { Friend.max(:age) }.should raise_error(ArgumentError)
-        lambda { Friend.avg(:age) }.should raise_error(ArgumentError)
-        lambda { Friend.sum(:age) }.should raise_error(ArgumentError)
+        lambda { Friend.min(:age) }.should raise_error(NotImplementedError)
+        lambda { Friend.max(:age) }.should raise_error(NotImplementedError)
+        lambda { Friend.avg(:age) }.should raise_error(NotImplementedError)
+        lambda { Friend.sum(:age) }.should raise_error(NotImplementedError)
       end
     end
   end