fake_dynamo 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # FakeDynamo
2
2
 
3
- local hosted, inmemory fake dynamodb
3
+ local hosted, inmemory dynamodb emulator.
4
4
 
5
5
 
6
6
  # Caveats
@@ -27,4 +27,4 @@ AWS.config(:use_ssl => false,
27
27
  ````
28
28
 
29
29
  # Storage
30
- fake_dynamo stores the `write commands` in `/usr/local/var/fake_dynamo/db.fdb` and replays it before starting.
30
+ fake_dynamo stores the `write operations` (request that changes the data) in `/usr/local/var/fake_dynamo/db.fdb` and replays it before starting the server.
data/bin/fake_dynamo CHANGED
@@ -4,15 +4,24 @@ $:.unshift(File.dirname(__FILE__) + '/../lib')
4
4
  require 'fake_dynamo'
5
5
  require 'optparse'
6
6
 
7
- options = {:port => 4567}
7
+ options = {:port => 4567, :compact => false }
8
8
  OptionParser.new do |opts|
9
9
  opts.banner = "Usage: fake_dynamo [options]"
10
10
 
11
11
  opts.on("-p", "--port PORT") do |v|
12
12
  options[:port] = v
13
13
  end
14
+
15
+ opts.on("-c", "--compact") do |v|
16
+ options[:compact] = v
17
+ end
14
18
  end.parse!
15
19
 
20
+ if options[:compact]
21
+ FakeDynamo::Storage.instance.load_aof
22
+ FakeDynamo::Storage.instance.compact!
23
+ end
24
+
16
25
  FakeDynamo::Storage.instance.load_aof
17
26
  FakeDynamo::Server.run!(:port => options[:port])
18
27
 
@@ -4,6 +4,20 @@ module FakeDynamo
4
4
 
5
5
  def initialize(name, value, type)
6
6
  @name, @value, @type = name, value, type
7
+
8
+ if ['NS', 'SS'].include? @type
9
+ raise ValidationException, 'Input collection contains duplicates' if value.uniq!
10
+ end
11
+
12
+ if ['NS', 'N'].include? @type
13
+ Array(@value).each do |n|
14
+ begin
15
+ Integer(n)
16
+ rescue
17
+ raise ValidationException, "The parameter cannot be converted to a numeric value: #{n}"
18
+ end
19
+ end
20
+ end
7
21
  end
8
22
 
9
23
  def description
@@ -3,7 +3,7 @@ module FakeDynamo
3
3
 
4
4
  include Validation
5
5
 
6
- attr_reader :tables
6
+ attr_accessor :tables
7
7
 
8
8
  class << self
9
9
  def instance
@@ -37,7 +37,7 @@ module FakeDynamo
37
37
 
38
38
  def validate_size(value_list, size)
39
39
  if (size.kind_of? Range and (not (size.include? value_list.size))) or
40
- (size.kind_of? Fixnum and value_list.size != size)
40
+ (size.kind_of? Integer and value_list.size != size)
41
41
  raise ValidationException, "The attempted filter operation is not supported for the provided filter argument count"
42
42
  end
43
43
  end
@@ -93,6 +93,7 @@ module FakeDynamo
93
93
  old_attribute.value = (old_attribute.value.to_i + attribute.value.to_i).to_s
94
94
  else
95
95
  old_attribute.value += attribute.value
96
+ old_attribute.value.uniq!
96
97
  end
97
98
  else
98
99
  attributes[name] = attribute
@@ -1,6 +1,11 @@
1
+ require 'fileutils'
2
+ require 'tempfile'
3
+
1
4
  module FakeDynamo
2
5
  class Storage
3
6
 
7
+ attr_accessor :compacted, :loaded
8
+
4
9
  class << self
5
10
  def instance
6
11
  @storage ||= Storage.new
@@ -51,11 +56,13 @@ module FakeDynamo
51
56
  return unless write_command?(operation)
52
57
  db_aof.puts(operation)
53
58
  data = data.to_json
54
- db_aof.puts(data.size + 1)
59
+ db_aof.puts(data.bytesize + "\n".bytesize)
55
60
  db_aof.puts(data)
61
+ db_aof.flush
56
62
  end
57
63
 
58
64
  def load_aof
65
+ return if @loaded
59
66
  file = File.new(db_path, 'r')
60
67
  puts "Loading fake_dynamo data ..."
61
68
  loop do
@@ -66,6 +73,35 @@ module FakeDynamo
66
73
  end
67
74
  rescue EOFError
68
75
  file.close
76
+ compact_if_necessary
77
+ @loaded = true
78
+ end
79
+
80
+ def compact_threshold
81
+ 100 * 1024 * 1024 # 100mb
82
+ end
83
+
84
+ def compact_if_necessary
85
+ return unless File.exists? db_path
86
+ if File.stat(db_path).size > compact_threshold
87
+ compact!
88
+ end
89
+ end
90
+
91
+ def compact!
92
+ return if @compacted
93
+ @aof = Tempfile.new('compact')
94
+ puts "Compacting db ..."
95
+ db.tables.each do |_, table|
96
+ persist('CreateTable', table.create_table_data)
97
+ table.items.each do |_, item|
98
+ persist('PutItem', table.put_item_data(item))
99
+ end
100
+ end
101
+ @aof.close
102
+ FileUtils.mv(@aof.path, db_path)
103
+ @aof = nil
104
+ @compacted = true
69
105
  end
70
106
  end
71
107
  end
@@ -27,6 +27,24 @@ module FakeDynamo
27
27
  }
28
28
  end
29
29
 
30
+ def create_table_data
31
+ {
32
+ 'TableName' => name,
33
+ 'KeySchema' => key_schema.description,
34
+ 'ProvisionedThroughput' => {
35
+ 'ReadCapacityUnits' => read_capacity_units,
36
+ 'WriteCapacityUnits' => write_capacity_units
37
+ }
38
+ }
39
+ end
40
+
41
+ def put_item_data(item)
42
+ {
43
+ 'TableName' => name,
44
+ 'Item' => item.as_hash
45
+ }
46
+ end
47
+
30
48
  def size_description
31
49
  { 'ItemCount' => items.count,
32
50
  'TableSizeBytes' => size_bytes }
@@ -130,16 +148,31 @@ module FakeDynamo
130
148
  else
131
149
  return consumed_capacity
132
150
  end
151
+ item_created = true
133
152
  end
134
153
 
135
- old_hash = item.as_hash
136
- data['AttributeUpdates'].each do |name, update_data|
137
- item.update(name, update_data)
154
+ old_item = deep_copy(item)
155
+ begin
156
+ old_hash = item.as_hash
157
+ data['AttributeUpdates'].each do |name, update_data|
158
+ item.update(name, update_data)
159
+ end
160
+ rescue => e
161
+ if item_created
162
+ @items.delete(key)
163
+ else
164
+ @items[key] = old_item
165
+ end
166
+ raise e
138
167
  end
139
168
 
140
169
  consumed_capacity.merge(return_values(data, old_hash, item))
141
170
  end
142
171
 
172
+ def deep_copy(x)
173
+ Marshal.load(Marshal.dump(x))
174
+ end
175
+
143
176
  def query(data)
144
177
  unless key_schema.range_key
145
178
  raise ValidationException, "Query can be performed only on a table with a HASH,RANGE key schema"
@@ -52,9 +52,9 @@ module FakeDynamo
52
52
  when :string
53
53
  add_errors("The parameter '#{param(attribute, parents)}' must be a string") unless data.kind_of? String
54
54
  when :long
55
- add_errors("The parameter '#{param(attribute, parents)}' must be a long") unless data.kind_of? Fixnum
55
+ add_errors("The parameter '#{param(attribute, parents)}' must be a long") unless data.kind_of? Integer
56
56
  when :integer
57
- add_errors("The parameter '#{param(attribute, parents)}' must be a integer") unless data.kind_of? Fixnum
57
+ add_errors("The parameter '#{param(attribute, parents)}' must be a integer") unless data.kind_of? Integer
58
58
  when :boolean
59
59
  add_errors("The parameter '#{param(attribute, parents)}' must be a boolean") unless (data.kind_of? TrueClass or data.kind_of? FalseClass)
60
60
  when Hash
@@ -1,3 +1,3 @@
1
1
  module FakeDynamo
2
- VERSION = "0.0.2"
2
+ VERSION = "0.0.3"
3
3
  end
@@ -24,7 +24,6 @@ module FakeDynamo
24
24
  it 'tests le' do
25
25
  subject.le_filter([{'S' => 'c'}], s_attr, false).should be_true
26
26
  subject.le_filter([{'S' => 'bcd'}], s_attr, false).should be_true
27
- subject.le_filter([{'N' => 'bcd'}], s_attr, false).should be_false
28
27
  subject.le_filter([{'S' => 'a'}], s_attr, false).should be_false
29
28
  subject.le_filter([{'N' => '10'}], n_attr, false).should be_true
30
29
  subject.le_filter([{'N' => '11'}], n_attr, false).should be_true
@@ -34,7 +33,6 @@ module FakeDynamo
34
33
  it 'tests lt' do
35
34
  subject.lt_filter([{'S' => 'c'}], s_attr, false).should be_true
36
35
  subject.lt_filter([{'S' => 'bcd'}], s_attr, false).should be_false
37
- subject.lt_filter([{'N' => 'bcd'}], s_attr, false).should be_false
38
36
  subject.lt_filter([{'S' => 'a'}], s_attr, false).should be_false
39
37
  subject.lt_filter([{'N' => '10'}], n_attr, false).should be_false
40
38
  subject.lt_filter([{'N' => '11'}], n_attr, false).should be_true
@@ -44,7 +42,6 @@ module FakeDynamo
44
42
  it 'test ge' do
45
43
  subject.ge_filter([{'S' => 'c'}], s_attr, false).should be_false
46
44
  subject.ge_filter([{'S' => 'bcd'}], s_attr, false).should be_true
47
- subject.ge_filter([{'N' => 'bcd'}], s_attr, false).should be_false
48
45
  subject.ge_filter([{'S' => 'a'}], s_attr, false).should be_true
49
46
  subject.ge_filter([{'N' => '10'}], n_attr, false).should be_true
50
47
  subject.ge_filter([{'N' => '11'}], n_attr, false).should be_false
@@ -54,7 +51,6 @@ module FakeDynamo
54
51
  it 'test gt' do
55
52
  subject.gt_filter([{'S' => 'c'}], s_attr, false).should be_false
56
53
  subject.gt_filter([{'S' => 'bcd'}], s_attr, false).should be_false
57
- subject.gt_filter([{'N' => 'bcd'}], s_attr, false).should be_false
58
54
  subject.gt_filter([{'S' => 'a'}], s_attr, false).should be_true
59
55
  subject.gt_filter([{'N' => '10'}], n_attr, false).should be_false
60
56
  subject.gt_filter([{'N' => '11'}], n_attr, false).should be_false
@@ -82,6 +82,12 @@ module FakeDynamo
82
82
  subject.attributes['set'].value.should eq(['1', '2', '3'])
83
83
  end
84
84
 
85
+ it "should handle duplicate in sets" do
86
+ subject.attributes['set'] = Attribute.new('set', ['1', '2'], 'SS')
87
+ subject.add('set', { 'SS' => ['3', '2']})
88
+ subject.attributes['set'].value.should eq(['1', '2', '3'])
89
+ end
90
+
85
91
  it "should handle type mismatch" do
86
92
  subject.attributes['xxx'] = Attribute.new('xxx', ['1', '2'], 'NS')
87
93
  expect { subject.add('xxx', {'SS' => ['3']}) }.to raise_error(ValidationException, /type mismatch/i)
@@ -0,0 +1,45 @@
1
+ # -*- coding: utf-8 -*-
2
+ require 'spec_helper'
3
+
4
+ module FakeDynamo
5
+ describe Storage do
6
+
7
+ let(:table) do
8
+ {"TableName" => "User",
9
+ "KeySchema" =>
10
+ {"HashKeyElement" => {"AttributeName" => "id","AttributeType" => "S"}},
11
+ "ProvisionedThroughput" => {"ReadCapacityUnits" => 5,"WriteCapacityUnits" => 10}
12
+ }
13
+ end
14
+
15
+ def item(i)
16
+ {'TableName' => 'User',
17
+ 'Item' => { 'id' => { 'S' => (i % 100).to_s },
18
+ 'name' => { 'S' => "╩tr¥in" }}
19
+ }
20
+ end
21
+
22
+ it 'compacts and loads db properly' do
23
+ db = DB.instance
24
+ db.tables = {}
25
+
26
+ db.process('CreateTable', table)
27
+ subject.persist('CreateTable', table)
28
+
29
+ 1000.times do |i|
30
+ db.process('PutItem', item(i))
31
+ subject.persist('PutItem', item(i))
32
+ end
33
+
34
+ @items = db.tables.values.map { |t| t.items.values.map(&:as_hash) }
35
+ 3.times do
36
+ db.tables = {}
37
+ subject.loaded = false
38
+ subject.load_aof
39
+ subject.compacted = false
40
+ subject.compact!
41
+ db.tables.values.map { |t| t.items.values.map(&:as_hash) }.should == @items
42
+ end
43
+ end
44
+ end
45
+ end
@@ -46,7 +46,7 @@ module FakeDynamo
46
46
 
47
47
  its(:read_capacity_units) { should == 10 }
48
48
  its(:write_capacity_units) { should == 15 }
49
- its(:last_increased_time) { should be_a_kind_of(Fixnum) }
49
+ its(:last_increased_time) { should be_a_kind_of(Integer) }
50
50
  its(:last_decreased_time) { should be_nil }
51
51
  end
52
52
 
@@ -60,6 +60,37 @@ module FakeDynamo
60
60
  end.to raise_error(ValidationException, /missing.*item/i)
61
61
  end
62
62
 
63
+ it 'should fail if sets contains duplicates' do
64
+ expect do
65
+ subject.put_item({ 'TableName' => 'Table1',
66
+ 'Item' => {
67
+ 'AttributeName1' => { 'S' => "test" },
68
+ 'AttributeName2' => { 'N' => "3" },
69
+ 'AttributeName3' => { 'NS' => ["1", "3", "3"] }
70
+ }})
71
+ end.to raise_error(ValidationException, /duplicate/)
72
+ end
73
+
74
+ it 'should fail if value is of different type' do
75
+ expect do
76
+ subject.put_item({ 'TableName' => 'Table1',
77
+ 'Item' => {
78
+ 'AttributeName1' => { 'S' => "test" },
79
+ 'AttributeName2' => { 'N' => "3" },
80
+ 'AttributeName3' => { 'NS' => ["1", "3", "one"] }
81
+ }})
82
+ end.to raise_error(ValidationException, /numeric/)
83
+
84
+ expect do
85
+ subject.put_item({ 'TableName' => 'Table1',
86
+ 'Item' => {
87
+ 'AttributeName1' => { 'S' => "test" },
88
+ 'AttributeName2' => { 'N' => "3" },
89
+ 'AttributeName3' => { 'N' => "one" }
90
+ }})
91
+ end.to raise_error(ValidationException, /numeric/)
92
+ end
93
+
63
94
  it 'should fail if range key is not present' do
64
95
  expect do
65
96
  subject.put_item({ 'TableName' => 'Table1',
@@ -215,6 +246,22 @@ module FakeDynamo
215
246
  {'AttributeUpdates' => {'AttributeName3' => {'Action' => 'DELETE'}}}
216
247
  end
217
248
 
249
+ it "should not partially update item" do
250
+ expect do
251
+ put['AttributeUpdates'].merge!({ 'xx' => { 'Value' => { 'N' => 'one'}, 'Action' => 'ADD'}})
252
+ subject.update_item(key.merge(put))
253
+ end.to raise_error(ValidationException, /numeric/)
254
+ subject.get_item(key).should include('Item' => item['Item'])
255
+
256
+ expect do
257
+ key['Key']['HashKeyElement']['S'] = 'unknown'
258
+ put['AttributeUpdates'].merge!({ 'xx' => { 'Value' => { 'N' => 'one'}, 'Action' => 'ADD'}})
259
+ subject.update_item(key.merge(put))
260
+ end.to raise_error(ValidationException, /numeric/)
261
+
262
+ subject.get_item(key).should eq(consumed_capacity)
263
+ end
264
+
218
265
  it "should check conditions" do
219
266
  expect do
220
267
  subject.update_item(key.merge({'Expected' =>
data/spec/spec_helper.rb CHANGED
@@ -1,7 +1,7 @@
1
1
  $: << File.join(File.dirname(File.dirname(__FILE__)), "lib")
2
2
 
3
3
  require 'simplecov'
4
- SimpleCov.start
4
+ SimpleCov.start if ENV['COVERAGE']
5
5
 
6
6
  require 'rspec'
7
7
  require 'rack/test'
@@ -22,7 +22,7 @@ module FakeDynamo
22
22
  end
23
23
 
24
24
  def db_path
25
- '/usr/local/var/fake_dynamo/test_db.fdb'
25
+ '/tmp/test_db.fdb'
26
26
  end
27
27
  end
28
28
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fake_dynamo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-03-13 00:00:00.000000000 Z
12
+ date: 2012-03-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sinatra
16
- requirement: &70152603384900 !ruby/object:Gem::Requirement
16
+ requirement: &70284887961460 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: '0'
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70152603384900
24
+ version_requirements: *70284887961460
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: activesupport
27
- requirement: &70152603384080 !ruby/object:Gem::Requirement
27
+ requirement: &70284887960600 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: '0'
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70152603384080
35
+ version_requirements: *70284887960600
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: json
38
- requirement: &70152603383260 !ruby/object:Gem::Requirement
38
+ requirement: &70284887955400 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,7 +43,7 @@ dependencies:
43
43
  version: '0'
44
44
  type: :runtime
45
45
  prerelease: false
46
- version_requirements: *70152603383260
46
+ version_requirements: *70284887955400
47
47
  description:
48
48
  email:
49
49
  - ananthakumaran@gmail.com
@@ -80,6 +80,7 @@ files:
80
80
  - spec/fake_dynamo/filter_spec.rb
81
81
  - spec/fake_dynamo/item_spec.rb
82
82
  - spec/fake_dynamo/server_spec.rb
83
+ - spec/fake_dynamo/storage_spec.rb
83
84
  - spec/fake_dynamo/table_spec.rb
84
85
  - spec/fake_dynamo/validation_spec.rb
85
86
  - spec/spec_helper.rb
@@ -112,6 +113,8 @@ test_files:
112
113
  - spec/fake_dynamo/filter_spec.rb
113
114
  - spec/fake_dynamo/item_spec.rb
114
115
  - spec/fake_dynamo/server_spec.rb
116
+ - spec/fake_dynamo/storage_spec.rb
115
117
  - spec/fake_dynamo/table_spec.rb
116
118
  - spec/fake_dynamo/validation_spec.rb
117
119
  - spec/spec_helper.rb
120
+ has_rdoc: