fake_dynamo 0.0.2 → 0.0.3

data/README.md

@@ -1,6 +1,6 @@
 # FakeDynamo
 
-local hosted, inmemory fake dynamodb
+local hosted, inmemory dynamodb emulator.
 
 
 # Caveats
@@ -27,4 +27,4 @@ AWS.config(:use_ssl => false,
 ````
 
 # Storage
-fake_dynamo stores the `write commands` in `/usr/local/var/fake_dynamo/db.fdb` and replays it before starting.
+fake_dynamo stores the `write operations` (request that changes the data) in `/usr/local/var/fake_dynamo/db.fdb` and replays it before starting the server.

data/bin/fake_dynamo

@@ -4,15 +4,24 @@ $:.unshift(File.dirname(__FILE__) + '/../lib')
 require 'fake_dynamo'
 require 'optparse'
 
-options = {:port => 4567}
+options = {:port => 4567, :compact => false }
 OptionParser.new do |opts|
   opts.banner = "Usage: fake_dynamo [options]"
 
   opts.on("-p", "--port PORT") do |v|
     options[:port] = v
   end
+
+  opts.on("-c", "--compact") do |v|
+    options[:compact] = v
+  end
 end.parse!
 
+if options[:compact]
+  FakeDynamo::Storage.instance.load_aof
+  FakeDynamo::Storage.instance.compact!
+end
+
 FakeDynamo::Storage.instance.load_aof
 FakeDynamo::Server.run!(:port => options[:port])
 

data/lib/fake_dynamo/attribute.rb

@@ -4,6 +4,20 @@ module FakeDynamo
 
     def initialize(name, value, type)
       @name, @value, @type = name, value, type
+
+      if ['NS', 'SS'].include? @type
+        raise ValidationException, 'Input collection contains duplicates' if value.uniq!
+      end
+
+      if ['NS', 'N'].include? @type
+        Array(@value).each do |n|
+          begin
+            Integer(n)
+          rescue
+            raise ValidationException, "The parameter cannot be converted to a numeric value: #{n}"
+          end
+        end
+      end
     end
 
     def description

data/lib/fake_dynamo/db.rb

@@ -3,7 +3,7 @@ module FakeDynamo
 
     include Validation
 
-    attr_reader :tables
+    attr_accessor :tables
 
     class << self
       def instance

data/lib/fake_dynamo/filter.rb

@@ -37,7 +37,7 @@ module FakeDynamo
 
     def validate_size(value_list, size)
       if (size.kind_of? Range and (not (size.include? value_list.size))) or
-          (size.kind_of? Fixnum and value_list.size != size)
+          (size.kind_of? Integer and value_list.size != size)
         raise ValidationException, "The attempted filter operation is not supported for the provided filter argument count"
       end
     end

data/lib/fake_dynamo/item.rb

@@ -93,6 +93,7 @@ module FakeDynamo
           old_attribute.value = (old_attribute.value.to_i + attribute.value.to_i).to_s
         else
           old_attribute.value += attribute.value
+          old_attribute.value.uniq!
         end
       else
         attributes[name] = attribute

data/lib/fake_dynamo/storage.rb

@@ -1,6 +1,11 @@
+require 'fileutils'
+require 'tempfile'
+
 module FakeDynamo
   class Storage
 
+    attr_accessor :compacted, :loaded
+
     class << self
       def instance
         @storage ||= Storage.new
@@ -51,11 +56,13 @@ module FakeDynamo
       return unless write_command?(operation)
       db_aof.puts(operation)
       data = data.to_json
-      db_aof.puts(data.size + 1)
+      db_aof.puts(data.bytesize + "\n".bytesize)
       db_aof.puts(data)
+      db_aof.flush
     end
 
     def load_aof
+      return if @loaded
       file = File.new(db_path, 'r')
       puts "Loading fake_dynamo data ..."
       loop do
@@ -66,6 +73,35 @@ module FakeDynamo
       end
     rescue EOFError
       file.close
+      compact_if_necessary
+      @loaded = true
+    end
+
+    def compact_threshold
+      100 * 1024 * 1024 # 100mb
+    end
+
+    def compact_if_necessary
+      return unless File.exists? db_path
+      if File.stat(db_path).size > compact_threshold
+        compact!
+      end
+    end
+
+    def compact!
+      return if @compacted
+      @aof = Tempfile.new('compact')
+      puts "Compacting db ..."
+      db.tables.each do |_, table|
+        persist('CreateTable', table.create_table_data)
+        table.items.each do |_, item|
+          persist('PutItem', table.put_item_data(item))
+        end
+      end
+      @aof.close
+      FileUtils.mv(@aof.path, db_path)
+      @aof = nil
+      @compacted = true
     end
   end
 end

data/lib/fake_dynamo/table.rb

@@ -27,6 +27,24 @@ module FakeDynamo
       }
     end
 
+    def create_table_data
+      {
+        'TableName' => name,
+        'KeySchema' => key_schema.description,
+        'ProvisionedThroughput' => {
+          'ReadCapacityUnits' => read_capacity_units,
+          'WriteCapacityUnits' => write_capacity_units
+        }
+      }
+    end
+
+    def put_item_data(item)
+      {
+        'TableName' => name,
+        'Item' => item.as_hash
+      }
+    end
+
     def size_description
       { 'ItemCount' => items.count,
         'TableSizeBytes' => size_bytes }
@@ -130,16 +148,31 @@ module FakeDynamo
         else
           return consumed_capacity
         end
+        item_created = true
       end
 
-      old_hash = item.as_hash
-      data['AttributeUpdates'].each do |name, update_data|
-        item.update(name, update_data)
+      old_item = deep_copy(item)
+      begin
+        old_hash = item.as_hash
+        data['AttributeUpdates'].each do |name, update_data|
+          item.update(name, update_data)
+        end
+      rescue => e
+        if item_created
+          @items.delete(key)
+        else
+          @items[key] = old_item
+        end
+        raise e
       end
 
       consumed_capacity.merge(return_values(data, old_hash, item))
     end
 
+    def deep_copy(x)
+      Marshal.load(Marshal.dump(x))
+    end
+
     def query(data)
       unless key_schema.range_key
         raise ValidationException, "Query can be performed only on a table with a HASH,RANGE key schema"

data/lib/fake_dynamo/validation.rb

@@ -52,9 +52,9 @@ module FakeDynamo
         when :string
           add_errors("The parameter '#{param(attribute, parents)}' must be a string") unless data.kind_of? String
         when :long
-          add_errors("The parameter '#{param(attribute, parents)}' must be a long") unless data.kind_of? Fixnum
+          add_errors("The parameter '#{param(attribute, parents)}' must be a long") unless data.kind_of? Integer
         when :integer
-          add_errors("The parameter '#{param(attribute, parents)}' must be a integer") unless data.kind_of? Fixnum
+          add_errors("The parameter '#{param(attribute, parents)}' must be a integer") unless data.kind_of? Integer
         when :boolean
           add_errors("The parameter '#{param(attribute, parents)}' must be a boolean") unless (data.kind_of? TrueClass or data.kind_of? FalseClass)
         when Hash

data/lib/fake_dynamo/version.rb

@@ -1,3 +1,3 @@
 module FakeDynamo
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spec/fake_dynamo/filter_spec.rb

@@ -24,7 +24,6 @@ module FakeDynamo
     it 'tests le' do
       subject.le_filter([{'S' => 'c'}], s_attr, false).should be_true
       subject.le_filter([{'S' => 'bcd'}], s_attr, false).should be_true
-      subject.le_filter([{'N' => 'bcd'}], s_attr, false).should be_false
       subject.le_filter([{'S' => 'a'}], s_attr, false).should be_false
       subject.le_filter([{'N' => '10'}], n_attr, false).should be_true
       subject.le_filter([{'N' => '11'}], n_attr, false).should be_true
@@ -34,7 +33,6 @@ module FakeDynamo
     it 'tests lt' do
       subject.lt_filter([{'S' => 'c'}], s_attr, false).should be_true
       subject.lt_filter([{'S' => 'bcd'}], s_attr, false).should be_false
-      subject.lt_filter([{'N' => 'bcd'}], s_attr, false).should be_false
       subject.lt_filter([{'S' => 'a'}], s_attr, false).should be_false
       subject.lt_filter([{'N' => '10'}], n_attr, false).should be_false
       subject.lt_filter([{'N' => '11'}], n_attr, false).should be_true
@@ -44,7 +42,6 @@ module FakeDynamo
     it 'test ge' do
       subject.ge_filter([{'S' => 'c'}], s_attr, false).should be_false
       subject.ge_filter([{'S' => 'bcd'}], s_attr, false).should be_true
-      subject.ge_filter([{'N' => 'bcd'}], s_attr, false).should be_false
       subject.ge_filter([{'S' => 'a'}], s_attr, false).should be_true
       subject.ge_filter([{'N' => '10'}], n_attr, false).should be_true
       subject.ge_filter([{'N' => '11'}], n_attr, false).should be_false
@@ -54,7 +51,6 @@ module FakeDynamo
     it 'test gt' do
       subject.gt_filter([{'S' => 'c'}], s_attr, false).should be_false
       subject.gt_filter([{'S' => 'bcd'}], s_attr, false).should be_false
-      subject.gt_filter([{'N' => 'bcd'}], s_attr, false).should be_false
       subject.gt_filter([{'S' => 'a'}], s_attr, false).should be_true
       subject.gt_filter([{'N' => '10'}], n_attr, false).should be_false
       subject.gt_filter([{'N' => '11'}], n_attr, false).should be_false

data/spec/fake_dynamo/item_spec.rb

@@ -82,6 +82,12 @@ module FakeDynamo
         subject.attributes['set'].value.should eq(['1', '2', '3'])
       end
 
+      it "should handle duplicate in sets" do
+        subject.attributes['set'] = Attribute.new('set', ['1', '2'], 'SS')
+        subject.add('set', { 'SS' => ['3', '2']})
+        subject.attributes['set'].value.should eq(['1', '2', '3'])
+      end
+
       it "should handle type mismatch" do
         subject.attributes['xxx'] = Attribute.new('xxx', ['1', '2'], 'NS')
         expect { subject.add('xxx', {'SS' => ['3']}) }.to raise_error(ValidationException, /type mismatch/i)

data/spec/fake_dynamo/storage_spec.rb

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+module FakeDynamo
+  describe Storage do
+
+    let(:table) do
+      {"TableName" => "User",
+        "KeySchema" =>
+        {"HashKeyElement" => {"AttributeName" => "id","AttributeType" => "S"}},
+        "ProvisionedThroughput" => {"ReadCapacityUnits" => 5,"WriteCapacityUnits" => 10}
+      }
+    end
+
+    def item(i)
+      {'TableName' => 'User',
+        'Item' => { 'id' => { 'S' => (i % 100).to_s },
+          'name' => { 'S' => "╩tr¥in" }}
+      }
+    end
+
+    it 'compacts and loads db properly' do
+      db = DB.instance
+      db.tables = {}
+
+      db.process('CreateTable', table)
+      subject.persist('CreateTable', table)
+
+      1000.times do |i|
+        db.process('PutItem', item(i))
+        subject.persist('PutItem', item(i))
+      end
+
+      @items = db.tables.values.map { |t| t.items.values.map(&:as_hash) }
+      3.times do
+        db.tables = {}
+        subject.loaded = false
+        subject.load_aof
+        subject.compacted = false
+        subject.compact!
+        db.tables.values.map { |t| t.items.values.map(&:as_hash) }.should == @items
+      end
+    end
+  end
+end

data/spec/fake_dynamo/table_spec.rb

@@ -46,7 +46,7 @@ module FakeDynamo
 
       its(:read_capacity_units) { should == 10 }
       its(:write_capacity_units) { should == 15 }
-      its(:last_increased_time) { should be_a_kind_of(Fixnum) }
+      its(:last_increased_time) { should be_a_kind_of(Integer) }
       its(:last_decreased_time) { should be_nil }
     end
 
@@ -60,6 +60,37 @@ module FakeDynamo
         end.to raise_error(ValidationException, /missing.*item/i)
       end
 
+      it 'should fail if sets contains duplicates' do
+        expect do
+          subject.put_item({ 'TableName' => 'Table1',
+                             'Item' => {
+                               'AttributeName1' => { 'S' => "test" },
+                               'AttributeName2' => { 'N' => "3" },
+                               'AttributeName3' => { 'NS' => ["1", "3", "3"] }
+                             }})
+        end.to raise_error(ValidationException, /duplicate/)
+      end
+
+      it 'should fail if value is of different type' do
+        expect do
+          subject.put_item({ 'TableName' => 'Table1',
+                             'Item' => {
+                               'AttributeName1' => { 'S' => "test" },
+                               'AttributeName2' => { 'N' => "3" },
+                               'AttributeName3' => { 'NS' => ["1", "3", "one"] }
+                             }})
+        end.to raise_error(ValidationException, /numeric/)
+
+        expect do
+          subject.put_item({ 'TableName' => 'Table1',
+                             'Item' => {
+                               'AttributeName1' => { 'S' => "test" },
+                               'AttributeName2' => { 'N' => "3" },
+                               'AttributeName3' => { 'N' => "one" }
+                             }})
+        end.to raise_error(ValidationException, /numeric/)
+      end
+
       it 'should fail if range key is not present' do
         expect do
           subject.put_item({ 'TableName' => 'Table1',
@@ -215,6 +246,22 @@ module FakeDynamo
         {'AttributeUpdates' => {'AttributeName3' => {'Action' => 'DELETE'}}}
       end
 
+      it "should not partially update item" do
+        expect do
+          put['AttributeUpdates'].merge!({ 'xx' => { 'Value' => { 'N' => 'one'}, 'Action' => 'ADD'}})
+          subject.update_item(key.merge(put))
+        end.to raise_error(ValidationException, /numeric/)
+        subject.get_item(key).should include('Item' => item['Item'])
+
+        expect do
+          key['Key']['HashKeyElement']['S'] = 'unknown'
+          put['AttributeUpdates'].merge!({ 'xx' => { 'Value' => { 'N' => 'one'}, 'Action' => 'ADD'}})
+          subject.update_item(key.merge(put))
+        end.to raise_error(ValidationException, /numeric/)
+
+        subject.get_item(key).should eq(consumed_capacity)
+      end
+
       it "should check conditions" do
         expect do
           subject.update_item(key.merge({'Expected' =>

data/spec/spec_helper.rb

@@ -1,7 +1,7 @@
 $: << File.join(File.dirname(File.dirname(__FILE__)), "lib")
 
 require 'simplecov'
-SimpleCov.start
+SimpleCov.start if ENV['COVERAGE']
 
 require 'rspec'
 require 'rack/test'
@@ -22,7 +22,7 @@ module FakeDynamo
     end
 
     def db_path
-      '/usr/local/var/fake_dynamo/test_db.fdb'
+      '/tmp/test_db.fdb'
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fake_dynamo
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-13 00:00:00.000000000 Z
+date: 2012-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70152603384900 !ruby/object:Gem::Requirement
+  requirement: &70284887961460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70152603384900
+  version_requirements: *70284887961460
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70152603384080 !ruby/object:Gem::Requirement
+  requirement: &70284887960600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70152603384080
+  version_requirements: *70284887960600
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70152603383260 !ruby/object:Gem::Requirement
+  requirement: &70284887955400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70152603383260
+  version_requirements: *70284887955400
 description: 
 email:
 - ananthakumaran@gmail.com
@@ -80,6 +80,7 @@ files:
 - spec/fake_dynamo/filter_spec.rb
 - spec/fake_dynamo/item_spec.rb
 - spec/fake_dynamo/server_spec.rb
+- spec/fake_dynamo/storage_spec.rb
 - spec/fake_dynamo/table_spec.rb
 - spec/fake_dynamo/validation_spec.rb
 - spec/spec_helper.rb
@@ -112,6 +113,8 @@ test_files:
 - spec/fake_dynamo/filter_spec.rb
 - spec/fake_dynamo/item_spec.rb
 - spec/fake_dynamo/server_spec.rb
+- spec/fake_dynamo/storage_spec.rb
 - spec/fake_dynamo/table_spec.rb
 - spec/fake_dynamo/validation_spec.rb
 - spec/spec_helper.rb
+has_rdoc: