dynameek 0.2.2 → 0.3.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dynameek (0.2.1)
+    dynameek (0.2.2)
       aws-sdk
 
 GEM
@@ -22,7 +22,7 @@ GEM
     i18n (0.6.1)
     json (1.7.7)
     multi_json (1.7.1)
-    nokogiri (1.5.8)
+    nokogiri (1.5.9)
     rack (1.5.2)
     rack-protection (1.5.0)
       rack

data/lib/dynameek/model/dynamo_db.rb

@@ -23,20 +23,38 @@ module Dynameek
             value
         end
       end
-    
+      
+      def index_table_exists?
+        !index_table.nil? && index_table.exists?
+      end
+
       def exists?
         table.exists?
       end
     
       def build!
-        return if dynamo_db.tables[table_name].exists?
+        return if dynamo_db.tables[table_name].exists?         
         opts = {:hash_key => { hash_key_info.field => [:datetime, :integer, :float].include?(hash_key_info.type) ? :number : hash_key_info.type }}
         if range?
           opts[:range_key] = { range_info.field => [:datetime, :integer, :float].include?(range_info.type) ? :number : range_info.type } 
         end
-        new_table = dynamo_db.tables.create(table_name, read_units, write_units, opts)
-        puts "Creating table, this may take a few minutes"
-        while new_table.status == :creating
+        new_table = nil
+        idx_table = nil
+        if index_table? && !dynamo_db.tables[index_table_name].exists?
+          dynamo_db.tables.create(
+                          table_name+"_INDEX", read_units, 
+                          write_units, opts)  
+          new_table     = dynamo_db.tables.create(
+                          table_name, read_units, write_units, {
+                            hash_key: {
+                              hash_key_info.field.to_s+"_"+range_info.field.to_s => :string
+                            },
+                            range_key: {"dynameek_index" => :number}
+                          })
+        else
+          new_table = dynamo_db.tables.create(table_name, read_units, write_units, opts)
+        end
+        while new_table.status == :creating || (!idx_table.nil? && idx_table.status == :creating)
           sleep 1
         end
       end
@@ -52,7 +70,14 @@ module Dynameek
         @table.load_schema if !@table.schema_loaded?
         @table
       end
-    
+      def index_table
+        build!
+        @index_table ||= dynamo_db.tables[index_table_name]
+        if @index_table.exists? && !@index_table.schema_loaded?
+          @index_table.load_schema
+        end
+        @index_table
+      end
     end
   end
-end
+end

data/lib/dynameek/model/query.rb

@@ -28,7 +28,11 @@ module Dynameek
         def all
           run
         end
-      
+        
+        def size
+          all.size
+        end
+
         def each
           all.each do |item|
             yield(item)
@@ -83,13 +87,45 @@ module Dynameek
             range = range.reduce({}) {|memo, (key, val)| memo[RANGE_QUERY_MAP[key]] = @model.convert_to_dynamodb(@model.range_info.type, val); memo}
           end
           query_hash.merge!(range)
-          @model.table.items.query(query_hash).map{|item| 
-            @model.item_to_instance(item)
-          }
+          if @model.index_table?
+            rows = @model.index_table.items.query(query_hash).map do |item|
+              item_hsh = @model.aws_item_to_hash(item).reduce({}){|m, (k,v)| m[k.to_sym] = v; m}
+              
+              # Need to convert from then back to because of the datetimes being
+              # screwy as number formats
+              hsh_val = @model.convert_from_dynamodb(@model.hash_key_info.type, 
+                                               item_hsh[@model.hash_key_info.field])
+              rng_val = @model.convert_from_dynamodb(@model.range_info.type, 
+                                               item_hsh[@model.range_info.field])
+              hsh_val = @model.convert_to_dynamodb(@model.hash_key_info.type, hsh_val)
+              rng_val = @model.convert_to_dynamodb(@model.range_info.type, rng_val)
+              item_hash_key = [hsh_val, @model.multi_column_join, rng_val].join('')
+              query_hsh_2 = {
+                hash_value: item_hash_key,
+                range_value: (1 .. item_hsh[:current_range_val].to_i),
+                select: :all
+              }
+#              p "QUERYING FOR #{query_hsh_2}"
+#              p "TABLE CONTENTS"
+#              p "--------------"
+#              @model.table.items.each do |i|
+#                p i.inspect
+#              end
+              internal_rows = @model.table.items.query(query_hsh_2).map{|act_item|
+                @model.item_to_instance(act_item)
+              }
+              internal_rows
+            end.flatten
+            rows
+          else
+            @model.table.items.query(query_hash).map{|item| 
+              @model.item_to_instance(item)
+            }
+          end
         end
       end
     
-      [:query, :where, :all, :each, :each_with_index, :delete].each do |method|
+      [:query, :where, :all, :each, :each_with_index, :size, :delete].each do |method|
         define_method(method) do |*args|
           qc = QueryChain.new(self)
           args = [] if !args
@@ -99,4 +135,4 @@ module Dynameek
     
     end
   end
-end
+end

data/lib/dynameek/model/structure.rb

@@ -24,6 +24,11 @@ module Dynameek
         @range
       end
       
+      def index_table?
+        @uses_index_table|| false
+      end
+      
+
       def range?
         !range_info.field.nil?
       end
@@ -74,7 +79,7 @@ module Dynameek
         hash_key_info.type = fields[fieldname]
         define_method(:hash_key) { read_attribute(fieldname) }
       end
-    
+
       def multi_column_hash_key fieldnames
         fieldnames = fieldnames.map(&:to_sym)
         fieldnames.each{|f| check_field(f)}
@@ -97,6 +102,40 @@ module Dynameek
         check_field(fieldname)
         range_info.field = fieldname
         range_info.type = fields[fieldname]
+        define_method(:range) do
+          attributes[self.class.range_info.field]
+        end
+      end
+     
+      def aws_item_to_hash(item)
+        (item.is_a?(AWS::DynamoDB::Item) || item.is_a?(AWS::DynamoDB::ItemData) ? 
+         item.attributes.to_hash : 
+         item
+        )
+      end
+
+      def current_range(key, range_val)
+        return nil if !index_table?
+        key.join!(multi_column_join) if key.is_a?(Array)
+        item = index_table.items.query(
+          {
+            hash_value: key,
+            select: :all,
+            range_value: range_val
+          }).first
+        return 0 if item.nil?
+        item_hsh = aws_item_to_hash(item)    
+        item_hsh['current_range_val'].to_i 
+      end
+  
+      def use_index_table
+        define_method(:act_hash_key) do
+          [self.class.convert_to_dynamodb(self.class.hash_key_info.type, hash_key), 
+            self.class.multi_column_join,
+            self.class.convert_to_dynamodb(self.class.range_info.type, range)].join('')
+        end
+        field :dynameek_index, :number
+        @uses_index_table = true
       end
 
       def check_field(fieldname)
@@ -107,4 +146,4 @@ module Dynameek
   end
 end
 
-  
+  

data/lib/dynameek/model.rb

@@ -14,11 +14,38 @@ module Dynameek
         memo
       end
       self.class.before_save_callbacks.each{|method| self.send method}
-      self.class.table.batch_write(
-        :put => [
-          attribs
-        ]
-      )
+      if self.class.index_table?
+        rng = self.class.convert_to_dynamodb(
+          self.class.range_info.type, 
+          attributes[self.class.range_info.field]
+        )
+        curr_range = self.class.current_range(hash_key, rng) + 1
+        self.class.index_table.batch_write(
+          :put => [
+            {
+              self.class.hash_key_info.field => attribs[self.class.hash_key_info.field],
+              self.class.range_info.field => attribs[self.class.range_info.field],
+              current_range_val: curr_range
+            }    
+          ]
+        )
+        attribs[self.class.hash_key_info.field.to_s+"_"+self.class.range_info.field.to_s] = 
+           attribs[self.class.hash_key_info.field].to_s +
+           self.class.multi_column_join + 
+           attribs[self.class.range_info.field].to_s
+        attribs[:dynameek_index] = curr_range
+        self.class.table.batch_write(
+            :put => [
+            attribs
+          ]
+        )
+      else
+        self.class.table.batch_write(
+            :put => [
+            attribs
+          ]
+        )
+      end
       self
     end
     
@@ -37,12 +64,11 @@ module Dynameek
     def delete
       if self.class.range?
         range_val = self.class.convert_to_dynamodb(self.class.range_info.type, self.send(self.class.range_info.field))
-        #Rounding errors can be irritating here so if we have the actual item we'll use it's range_val, nope that makes things worse
-        # range_val = dynamo_item.range_value if !dynamo_item.nil?
-        # p "TRYING TO DELETE #{[[hash_key, range_val]]}.inspect"
-        # p "FINDING THAT THING: #{self.class.find(hash_key, self.send(self.class.range_info.field)).inspect}"
-        # p "VIA BATCH GET  #{self.class.table.batch_get(:all, [[hash_key, range_val]]).entries.inspect}"
-        self.class.table.batch_delete([[hash_key, range_val]])
+        if self.class.index_table?
+          self.class.table.batch_delete([[act_hash_key, attributes[:dynameek_index]]])
+        else
+          self.class.table.batch_delete([[hash_key, range_val]])
+        end
       else  
         self.class.table.batch_delete([hash_key])
       end
@@ -76,11 +102,17 @@ module Dynameek
       end
   
       def delete_table
-        table.delete
+
+        index_table.delete if dynamo_db.tables[index_table_name].exists?
+        table.delete if dynamo_db.tables[table_name].exists?
+        while dynamo_db.tables[table_name].exists? && 
+              dynamo_db.tables[index_table_name].exists?
+          sleep 1
+        end
       end
   
       def item_to_instance(item)
-        item_hsh = (item.is_a?(AWS::DynamoDB::Item) || item.is_a?(AWS::DynamoDB::ItemData) ? item.attributes.to_hash : item)
+        item_hsh = aws_item_to_hash(item)
         instance = self.new
         fields.each do |field, type|
           next if multi_column_hash_key? && field == hash_key_info.field
@@ -111,7 +143,11 @@ module Dynameek
       def table_name
         self.to_s
       end
-  
+      
+      def index_table_name
+        table_name + "_INDEX"
+      end
+
     end
   end
-end
+end

data/lib/dynameek/version.rb

@@ -1,3 +1,3 @@
 module Dynameek
-  VERSION = "0.2.2"
+  VERSION = "0.3.1"
 end

data/test/models/with_index_table.rb

@@ -0,0 +1,13 @@
+class WithIndexTable
+  include Dynameek::Model
+
+  field :client_id, :integer
+  field :channel_id, :string
+  field :advert_id, :integer
+  field :time, :datetime
+
+  multi_column_hash_key [:client_id, :channel_id]
+  range :time
+
+  use_index_table
+end

data/test/spec/spec_helper.rb

@@ -10,4 +10,5 @@ AWS.config(:use_ssl => false,
            :secret_access_key => "xxx")
 
 require File.join(File.dirname(__FILE__), *%w[.. models conversion])
-require File.join(File.dirname(__FILE__), *%w[.. models simple])
+require File.join(File.dirname(__FILE__), *%w[.. models with_index_table])
+require File.join(File.dirname(__FILE__), *%w[.. models simple])

data/test/spec/with_index_table_spec.rb

@@ -0,0 +1,47 @@
+require File.join(File.dirname(__FILE__), *%w[spec_helper])
+
+describe WithIndexTable do
+  before(:all) do
+    WithIndexTable.delete_table
+  end
+  before(:each) do
+    WithIndexTable.query([1, "google"]).delete
+  end
+  
+  it "should allow you to create a new row" do
+    con = nil
+    lambda{
+      con = WithIndexTable.create(:client_id => 1, :channel_id => "google", :advert_id=> 1, :time => DateTime.now)
+    }.should_not raise_error
+    con.hash_key.should == "1|google" 
+  end
+
+  it "should allow you to have multiple rows for the same hash and range" do
+    row1, row2 = nil
+    time = DateTime.now
+    lambda{
+      row1 = WithIndexTable.create(:client_id => 1, :channel_id => "google", 
+                                   :advert_id=> 1, :time => time)
+      row2 = WithIndexTable.create(:client_id => 1, :channel_id => "google", 
+                                   :advert_id=> 2, :time => time)
+    }.should_not raise_error
+    row1.hash_key.should == "1|google"
+    row2.hash_key.should == "1|google"
+  end
+
+  it "should allow you to query for the some hash and range" do
+    row1, row2 = nil
+    time = DateTime.now
+    lambda{
+      row1 = WithIndexTable.create(:client_id => 1, :channel_id => "google", 
+                                   :advert_id=> 1, :time => time)
+      row2 = WithIndexTable.create(:client_id => 1, :channel_id => "google", 
+                                   :advert_id=> 2, :time => time)
+    }.should_not raise_error
+    
+    query = WithIndexTable.query([1, "google"]).where(time, :eq)
+    query.size.should == 2
+  
+  end
+
+end

metadata

@@ -2,7 +2,7 @@
 name: dynameek
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors: 
 - Max Dupenois
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-20 00:00:00 Z
+date: 2013-04-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: aws-sdk
@@ -69,9 +69,11 @@ files:
 - lib/dynameek/version.rb
 - test/models/conversion.rb
 - test/models/simple.rb
+- test/models/with_index_table.rb
 - test/spec/conversion_spec.rb
 - test/spec/simple_spec.rb
 - test/spec/spec_helper.rb
+- test/spec/with_index_table_spec.rb
 homepage: http://github.com/maxdupenois/dynameek
 licenses: []
 
@@ -102,6 +104,8 @@ summary: Dynameek - A dynamodb model
 test_files: 
 - test/models/conversion.rb
 - test/models/simple.rb
+- test/models/with_index_table.rb
 - test/spec/conversion_spec.rb
 - test/spec/simple_spec.rb
 - test/spec/spec_helper.rb
+- test/spec/with_index_table_spec.rb