cloudwow-not_relational 0.1.0 → 0.1.2

data/lib/not_relational/{repository.rb → sdb_repository.rb}

@@ -1,6 +1,6 @@
 # THis class implements access to SDB 222.
 # =I am the walrus=
-  module NotRelational
+module NotRelational
   $KCODE = 'u'
   require "aws_sdb"
   require "not_relational/storage.rb"
@@ -9,77 +9,59 @@
 
   require File.dirname(__FILE__) +"/sdb_monkey_patch.rb"
   # THis class implements access to SDB.
-  class Repository
-    attr_accessor :use_cache
+  class SdbRepository
+    MAX_PAGE_SIZE=250#defined by sdb
     attr_accessor :storage
+
     def initialize(
-        domain_name_prefix,#MyDevPayApp
+        base_domain_name,#MyDevPayApp
         clob_bucket,
         aws_key_id,
         aws_secret_key,
         memcache_servers = nil  ,
-        a_storage=nil,
-        append_table_to_domain=true,
+        blob_storage=nil,
+        use_seperate_domain_per_model=true,
         options={}
                     
       )
-                
-      @use_cache=true
-      @storage=a_storage
-      @storage||=Storage.new(aws_key_id,aws_secret_key,memcache_servers)
-      @domain_name_prefix = domain_name_prefix
-      @clob_bucket = clob_bucket
-      @memcache_servers = memcache_servers
-      @aws_key_id = aws_key_id
-      @aws_secret_key = aws_secret_key
-      @append_table_to_domain=append_table_to_domain
+
       @logger = options[:logger]
       if !@logger
         @logger = Logger.new(STDOUT)
         @logger.level = Logger::ERROR
       end
+
+
+    
+      @base_domain_name = base_domain_name
+      @storage_bucket = clob_bucket
+      @memcache_servers = memcache_servers
+      @aws_key_id = aws_key_id
+      @aws_secret_key = aws_secret_key
+      @use_seperate_domain_per_model=use_seperate_domain_per_model
+      @storage=blob_storage
+
+      @use_cache=true
+      @use_cache=options[:use_cache] if options.has_key?(:use_cache)
+
+
+      @storage ||= Storage.new(aws_key_id,aws_secret_key,memcache_servers)
       @sdb=AwsSdb::Service.new(:access_key_id=>aws_key_id,:secret_access_key=>aws_secret_key,:url=>"http://sdb.amazonaws.com",:logger=>@logger)
       @session_cache=MemoryRepository.new
-    
-      # #create_domain()
-    end
-    #can you guess what this does?
-    def clear_session_cache
-      @session_cache.clear
-    end
-    def flatten_key(key)
-      if key.is_a?( Array)
-        flattened_key=""
-        key.each do |key_part|
-          flattened_key<<CGI.escape(key_part.to_s)+"/"
-        end
-        return flattened_key[0..-2]
-      else
-        return CGI.escape(key.to_s)
-      end
-    end
-    def make_cache_key(table_name,primary_key)
 
-      primary_key=flatten_key(primary_key)
-      primary_key="#{table_name}/#{primary_key}" unless @append_table_to_domain
-      return primary_key
-    end
-    def make_clob_key(table_name,primary_key,clob_name)
-      return "clobs/#{table_name}/#{flatten_key(primary_key)}/#{clob_name}"
-    end
-    def get_clob(table_name,primary_key,clob_name)
-      return @storage.get(@clob_bucket,make_clob_key(table_name,primary_key,clob_name))
-        
     end
+    
     def save(table_name, primary_key, attributes,index_descriptions)
+
       @session_cache.save(table_name,primary_key,attributes,index_descriptions)
+
       formatted_attributes={}
       attributes.each do |description,value|
         if value || description.value_type==:boolean
-          if description.is_clob
+          if description.is_text?
             @storage.put(
-              @clob_bucket,
-              make_clob_key(table_name,primary_key,description.name),
+              @storage_bucket,
+              make_storage_key(table_name,primary_key,description.name),
               value.to_s,
               {})
           else
@@ -87,7 +69,7 @@
           end
         end
       end
-      if !@append_table_to_domain
+      if !@use_seperate_domain_per_model
         formatted_attributes['metadata%%table_name'] = table_name
 
       end
@@ -95,34 +77,153 @@
       # put_into_cache(table_name, primary_key, formatted_attributes)
   
     end
-    #  def put_into_cache(table_name, primary_key, formatted_attributes)
-    #      if @use_cache
-    #      cacheItem=DomainModelCacheItem.new(table_name, primary_key, formatted_attributes)
-    #
-    #      yaml=cacheItem.ya2yaml(:syck_compatible => true)
-    #
-    #      @storage.put(
-    #        @clob_bucket,
-    #        make_cache_key(table_name,primary_key),
-    #        yaml ,
-    #        {})
-    #    end
-    #  end
+
+    def destroy(table_name, primary_key)
+      @session_cache.destroy(table_name,primary_key)
+      @sdb.delete_attributes(make_domain_name(table_name),make_cache_key(table_name, primary_key) )
+      #TODO destroy text
+    end
+
+   
+    # result will be an array of hashes. each hash is a set of attributes
+    def query(table_name,attribute_descriptions,options)
+
+      @logger.debug "query on table: #{table_name}"
+
+      if options.has_key?(:limit) and !options.has_key?(:order_by)
+        session_cache_result=@session_cache.query(table_name,attribute_descriptions,options)
+        if options[:limit]==session_cache_result.length
+          return session_cache_result
+        end
+      end
+    
+      the_query=build_query(table_name, attribute_descriptions, options)
+      @logger.debug "the query: #{the_query}"
+
+      max=MAX_PAGE_SIZE
+      if options[:limit]
+        max=options[:limit].to_i
+      end
+
+      page_size=max> MAX_PAGE_SIZE ? MAX_PAGE_SIZE : max
+      sdb_result,token=sdb_query_with_attributes(table_name,the_query,page_size)
+
+      while !(token.nil? || token.empty? || sdb_result.length>=max)
+        page_size=max- sdb_result.length
+        page_size=page_size> MAX_PAGE_SIZE ? MAX_PAGE_SIZE : page_size
+        partial_results,token=sdb_query_with_attributes(table_name,the_query,page_size,token)
+        sdb_result.merge!( partial_results)
+      end
+
+      result=[]
+      sdb_result.each{|primary_key,sdb_row|
+        attributes =parse_attributes(attribute_descriptions,sdb_row)
+        if attributes
+          result<<attributes
+        end
+      }
+
+      #ordering is handled in sdb now
+#      if options and options[:order_by]
+#        result.sort! do |a,b|
+#          a_value=a[options[:order_by]]
+#          b_value=b[options[:order_by]]
+#          if options[:order] && options[:order]!=:ascending
+#            if !a_value
+#              1
+#            else
+#              if b_value
+#                b_value <=> a_value
+#              else
+#                -1
+#              end
+#            end
+#          else
+#            if !b_value
+#              1
+#            else
+#              if a_value
+#                a_value <=> b_value
+#              else
+#                -1
+#              end
+#            end
+#          end
+#        end
+#      end
+      if options[:limit] && result.length>options[:limit]
+        result=result[0..(options[:limit]-1)]
+      end
+      return result
+    end
+def get_text(table_name,primary_key,clob_name)
+      return @storage.get(@storage_bucket,make_storage_key(table_name,primary_key,clob_name))
+
+    end
+
+
+    def find_one(table_name, primary_key,attribute_descriptions)#, non_clob_attribute_names, clob_attribute_names)
+      session_cache_result=@session_cache.find_one(table_name, make_cache_key(table_name,primary_key),attribute_descriptions)
+      return session_cache_result if session_cache_result
+      #    if @use_cache
+      #      yaml=@storage.get(@storage_bucket,make_cache_key(table_name,primary_key))
+      #      if yaml
+      #        result=YAML::load( yaml)
+      #        if result.respond_to?(:non_clob_attributes) && result.non_clob_attributes!=nil
+      #          return parse_attributes(attribute_descriptions, result.non_clob_attributes)
+      #        end
+      #
+      #      end
+      #    end
+      attributes=parse_attributes(attribute_descriptions,sdb_get_attributes(table_name,primary_key))
+      if attributes
+        # #attributes[:primary_key]=primary_key #put_into_cache(table_name,
+        # primary_key, attributes)
+      end
+   
+      attributes
+    end
+
     def create_domain
       20.times do |i|
         begin
-          @sdb.create_domain(@domain_name_prefix)
+          @sdb.create_domain(@base_domain_name)
           return
-  
+
         rescue  => e
           s= "#{e.message}\n#{e.backtrace}"
           @logger.warn(s) if @logger
           sleep(i*i)
-               
+
         end
       end
-          
+
     end
+
+    def pause
+      sleep(2)
+    end
+
+    def  clear
+      @session_cache.clear
+    end
+
+    # can you guess what this does?
+    def clear_session_cache
+      @session_cache.clear
+    end
+
+    
+    private
+
+    def make_domain_name(table_name)
+      if @use_seperate_domain_per_model
+        @base_domain_name+"_"+table_name
+      else
+        @base_domain_name
+      end
+    end
+
     def put_attributes(table_name,primary_key, formatted_attributes,options={})
       20.times do |i|
         begin
@@ -132,17 +233,77 @@
         rescue Exception => e
           s= "#{e.message}\n#{e.backtrace}"
           @logger.warn(s) if @logger
-          puts e.to_yaml
           sleep(i*i)
         end
       end
+
+    end
+
+    def sdb_get_attributes(table_name,primary_key)
+      
+      @logger.debug( "SDB get_attributes #{table_name} : #{primary_key}") if @logger
+     
+      20.times do |i|
+        begin
+          return @sdb.get_attributes(make_domain_name(table_name), make_cache_key(table_name,primary_key))
+        rescue Exception => e
+          s= "#{e.message}\n#{e.backtrace}"
+          @logger.warn(s) if @logger
        
+          sleep(i*i)
+        ensure
+         
+        end
+      end
+    
     end
-    def extend_query(query,new_clause)
+    
+#    this method no longer useful since query with attrivutes was introduced to sdb
+#    def sdb_query(table_name,query,max,token=nil)
+#
+#      @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
+#      #      puts "#{table_name}  #{query}   (#{max}) #{token}"
+#      20.times do |i|
+#        begin
+#          return @sdb.query(make_domain_name(table_name),query,max,token)
+#
+#        rescue Exception => e
+#          s= "#{e.message}\n#{e.backtrace}"
+#          @logger.error(s) if @logger
+#
+#          sleep(i*i)
+#        ensure
+#
+#        end
+#      end
+#
+#    end
+
+    def sdb_query_with_attributes(table_name,query,max,token=nil)
+
+      @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
+      20.times do |i|
+        begin
+          return @sdb.query_with_attributes(make_domain_name(table_name),query,max,token)
+
+        rescue Exception => e
+          s= "#{e.message}\n#{e.backtrace}"
+          @logger.error(s) if @logger
+
+          sleep(i*i)
+        ensure
+
+        end
+      end
+
+    end
+
+
+     def extend_query(query,new_clause)
       if query.length>0
         query << " intersection "
       end
-    
+
       query << new_clause
     end
     def escape_quotes(value)
@@ -150,8 +311,8 @@
       value.gsub( "'","\\'")
     end
     def build_query(table_name,attribute_descriptions,options={})
-    
-    
+
+
       query=""
       params=nil
       params=options[:params] if options.has_key?(:params)
@@ -168,7 +329,7 @@
         end
 
       end
-      if !@append_table_to_domain
+      if !@use_seperate_domain_per_model
         extend_query(query," ['metadata%%table_name' = '#{table_name}']")
 
       end
@@ -226,161 +387,28 @@
             extend_query(query,"["+condition.to_sdb_query()+"]")
           end
         end
-             
-      end
-      return query
-   
-    end
-    # #result will be an array of hashes. each hash is a set of attributes
-    def query(table_name,attribute_descriptions,options)
-      if options.has_key?(:limit) and !options.has_key?(:order_by)
-        session_cache_result=@session_cache.query(table_name,attribute_descriptions,options)
-        if options[:limit]==session_cache_result.length
-          return session_cache_result
-        end
-      end
-    
-      the_query=build_query(table_name, attribute_descriptions, options)
-      max=250
-      if options[:limit]
-        max=options[:limit].to_i
-
-      end
-      page_size=max>250?250:max
-      sdb_result,token=sdb_query_with_attributes(table_name,the_query,page_size)
-
-      while !(token.nil? || token.empty? || sdb_result.length>=max)
-        page_size=max- sdb_result.length
-        page_size=page_size>250?250:page_size
-        partial_results,token=sdb_query_with_attributes(table_name,the_query,page_size,token)
 
-        sdb_result.merge!( partial_results)
       end
-      result=[]
-      sdb_result.each{|primary_key,sdb_row|
-        attributes =parse_attributes(attribute_descriptions,sdb_row)
-        if attributes
-          # #attributes[:primary_key]=primary_key
+      return query
 
-          result<<attributes
-        end
-      }
-      if options and options[:order_by]
-        result.sort! do |a,b|
-          a_value=a[options[:order_by]]
-          b_value=b[options[:order_by]]
-          if options[:order] && options[:order]!=:ascending
-            if !a_value
-              1
-            else
-              if b_value
-                b_value <=> a_value
-              else
-                -1
-              end
-            end
-          else
-            if !b_value
-              1
-            else
-              if a_value
-                a_value <=> b_value
-              else
-                -1
-              end
-            end
-          end
-        end
-      end
-      if options[:limit] && result.length>options[:limit]
-        result=result[0..(options[:limit]-1)]
-      end
-      return result
-    end
-    def find_one(table_name, primary_key,attribute_descriptions)#, non_clob_attribute_names, clob_attribute_names)
-      session_cache_result=@session_cache.find_one(table_name, make_cache_key(table_name,primary_key),attribute_descriptions)
-      return session_cache_result if session_cache_result
-      #    if @use_cache
-      #      yaml=@storage.get(@clob_bucket,make_cache_key(table_name,primary_key))
-      #      if yaml
-      #        result=YAML::load( yaml)
-      #        if result.respond_to?(:non_clob_attributes) && result.non_clob_attributes!=nil
-      #          return parse_attributes(attribute_descriptions, result.non_clob_attributes)
-      #        end
-      #
-      #      end
-      #    end
-      attributes=parse_attributes(attribute_descriptions,sdb_get_attributes(table_name,primary_key))
-      if attributes
-        # #attributes[:primary_key]=primary_key #put_into_cache(table_name,
-        # primary_key, attributes)
-      end
-   
-      attributes
-    end
-    def make_domain_name(table_name)
-      if @append_table_to_domain
-        @domain_name_prefix+"_"+table_name
-      else
-        @domain_name_prefix
-      end
-    end
-    def sdb_get_attributes(table_name,primary_key)
-      
-      @logger.debug( "SDB get_attributes #{table_name} : #{primary_key}") if @logger
-     
-      20.times do |i|
-        begin
-          return @sdb.get_attributes(make_domain_name(table_name), make_cache_key(table_name,primary_key))
-        rescue Exception => e
-          s= "#{e.message}\n#{e.backtrace}"
-          @logger.warn(s) if @logger
-       
-          sleep(i*i)
-        ensure
-         
-        end
-      end
-    
-    end
-    def sdb_query(table_name,query,max,token=nil)
-      
-      @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
-      #      puts "#{table_name}  #{query}   (#{max}) #{token}"
-      20.times do |i|
-        begin
-          return @sdb.query(make_domain_name(table_name),query,max,token)
-        
-        rescue Exception => e
-          s= "#{e.message}\n#{e.backtrace}"
-          @logger.error(s) if @logger
-       
-          sleep(i*i)
-        ensure
-         
-        end
-      end
-    
     end
-    def sdb_query_with_attributes(table_name,query,max,token=nil)
 
-      @logger.debug( "SDB query:#{table_name}(#{max}) : #{query}   #{token}"  ) if @logger
-      #      puts "#{table_name}  #{query}   (#{max}) #{token}"
-      20.times do |i|
-        begin
-          return @sdb.query_with_attributes(make_domain_name(table_name),query,max,token)
-
-        rescue Exception => e
-          s= "#{e.message}\n#{e.backtrace}"
-          @logger.error(s) if @logger
-
-          sleep(i*i)
-        ensure
+#  def put_into_cache(table_name, primary_key, formatted_attributes)
+    #      if @use_cache
+    #      cacheItem=DomainModelCacheItem.new(table_name, primary_key, formatted_attributes)
+    #
+    #      yaml=cacheItem.ya2yaml(:syck_compatible => true)
+    #
+    #      @storage.put(
+    #        @storage_bucket,
+    #        make_cache_key(table_name,primary_key),
+    #        yaml ,
+    #        {})
+    #    end
+    #  end
 
-        end
-      end
 
-    end
+    
     def parse_attributes(attribute_descriptions,attributes)
       if !attributes || attributes.length==0
         return nil
@@ -399,21 +427,31 @@
       end
       parsed_attributes
     end
-    def destroy(table_name, primary_key)
-      @session_cache.destroy(table_name,primary_key)
-      @sdb.delete_attributes(make_domain_name(table_name),make_cache_key(table_name, primary_key) )
-      #    if @use_cache
-      #      @storage.delete(@clob_bucket,make_cache_key(table_name,primary_key))
-      #    end
+
+    
+    def flatten_key(key)
+      if key.is_a?( Array)
+        flattened_key=""
+        key.each do |key_part|
+          flattened_key<<CGI.escape(key_part.to_s)+"/"
+        end
+        return flattened_key[0..-2]
+      else
+        return CGI.escape(key.to_s)
+      end
     end
-    # #parse date in yyyy-mm-dd format
 
-    def pause
-      sleep(2)
+    def make_cache_key(table_name,primary_key)
+
+      primary_key=flatten_key(primary_key)
+      primary_key="#{table_name}/#{primary_key}" unless @use_seperate_domain_per_model
+      return primary_key
     end
 
-    def  clear
-      @session_cache.clear
+    def make_storage_key(table_name,primary_key,clob_name)
+      return "clobs/#{table_name}/#{flatten_key(primary_key)}/#{clob_name}"
     end
+
+    
   end
 end  

data/lib/not_relational/storage.rb

@@ -72,7 +72,7 @@ class Storage
     if self.memory_only
       return nil
     end
-    20.times do |i|
+    4.times do |i|
       begin
         response=real_s3.get(bucket,key)
         if response and response.http_response.code=='404'
@@ -103,7 +103,7 @@ class Storage
 
     return if self.memory_only
 x=nil
-    20.times do |i|
+    4.times do |i|
       begin
       x= real_s3.put(bucket,key,S3::S3Object.new(object),attributes)
       

data/lib/not_relational.rb

@@ -5,7 +5,7 @@
 # License::   Distributes under the same terms as Ruby
 
 
-require "#{File.dirname(__FILE__)}/not_relational/repository.rb"
+require "#{File.dirname(__FILE__)}/not_relational/sdb_repository.rb"
 require "#{File.dirname(__FILE__)}/not_relational/repository_factory.rb"
 require "#{File.dirname(__FILE__)}/not_relational/acts_as_not_relational_application.rb"
 require "#{File.dirname(__FILE__)}/not_relational/memory_repository.rb"
@@ -14,4 +14,7 @@ require "#{File.dirname(__FILE__)}/not_relational/storage.rb"
 require "#{File.dirname(__FILE__)}/not_relational/s3.rb"
 require "#{File.dirname(__FILE__)}/not_relational/attribute_range.rb"
 require "#{File.dirname(__FILE__)}/not_relational/and_condition.rb"
-require "#{File.dirname(__FILE__)}/not_relational/or_condition.rb"
+require "#{File.dirname(__FILE__)}/not_relational/or_condition.rb"
+require "#{File.dirname(__FILE__)}/not_relational/starts_with_condition.rb"
+require "#{File.dirname(__FILE__)}/not_relational/is_null_transform.rb"
+require "#{File.dirname(__FILE__)}/not_relational/geo.rb"

data/test/models/album.rb

@@ -9,7 +9,7 @@ class Album < NotRelational::DomainModel
   property :guid , :string,:unique=>true
   property :user_name , :string
   property :title , :string
-  property :description , :clob 
+  property :description , :text 
   property :created_time_utc ,:date  
   property :last_update_time_utc ,:date  
   property :is_private, :boolean

data/test/models/blurb_wording.rb

@@ -7,7 +7,7 @@ class BlurbWording < NotRelational::DomainModel
   property :blurb_name,:string,:is_primary_key=>true
   property :blurb_namespace,:string,:is_primary_key=>true
   property :language_id , :string  ,:is_primary_key=>true
-  property :text,:clob
+  property :text,:text
   property :title , :string  
   property :version , :string  
   property :author , :string  

data/test/models/friend_request.rb

@@ -8,7 +8,7 @@ class FriendRequest < DomainModel
   property :id,:string,:is_primary_key=>true
    property :user_name,:string
    property :friend_user_name,:string
-   property :message,:clob
+   property :message,:text
    property :answer,:boolean
    property :created_time_utc,:date
  belongs_to :User,:user_name,:user

data/test/models/group.rb

@@ -15,7 +15,7 @@ class Group < NotRelational::DomainModel
   property :short_description , :string
   property :icon_guid , :string
   property :tags , :string
-  property :layout , :clob
+  property :layout , :text
   property :domain , :string,:unique=>true
   
   property :devpay_user_token , :string

data/test/models/media_item.rb

@@ -15,7 +15,7 @@ class Mediaitem < NotRelational::DomainModel
   property :group_id , :string
   property :title , :string
   property :user_login , :string
-  property :description , :clob  
+  property :description , :text  
   property :latitude        ,         :float                                                                                                                 
   property :longitude       ,         :float