sdbtools 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.3.0
1
+ 0.4.0
@@ -1,149 +1,15 @@
1
1
  require 'fattr'
2
2
  require 'right_aws'
3
- require File.expand_path('selection', File.dirname(__FILE__))
3
+ $:.unshift(File.expand_path(File.dirname(__FILE__)))
4
+ require 'sdbtools/operation'
5
+ require 'sdbtools/database'
6
+ require 'sdbtools/domain'
7
+ require 'sdbtools/selection'
8
+ require 'sdbtools/transaction'
9
+ require 'sdbtools/measured_sdb_interface'
4
10
 
5
11
  module SDBTools
6
12
 
7
- # An Operation represents a SimpleDB operation and handles the details of
8
- # re-requesting "next tokens" until the operation is complete.
9
- class Operation
10
- include Enumerable
11
-
12
- attr_reader :method
13
- attr_reader :args
14
- attr_reader :starting_token
15
-
16
- def initialize(sdb, method, *args)
17
- @options = args.last.is_a?(Hash) ? args.pop : {}
18
- @sdb = sdb
19
- @method = method
20
- @args = args
21
- @starting_token = @options[:starting_token]
22
- end
23
-
24
- # Yields once for each result set, until there is no next token.
25
- def each
26
- next_token = starting_token
27
- begin
28
- args = @args.dup
29
- args << next_token
30
- results = @sdb.send(@method, *args)
31
- yield(results)
32
- next_token = results[:next_token]
33
- end while next_token
34
- end
35
-
36
- end
37
-
38
- class Database
39
- def initialize(access_key=nil, secret_key=nil, options={})
40
- @sdb = RightAws::SdbInterface.new(access_key, secret_key, options)
41
- @logger = options.fetch(:logger){::Logger.new($stderr)}
42
- end
43
-
44
- def domains
45
- domains_op = Operation.new(@sdb, :list_domains, nil)
46
- domains_op.inject([]) {|domains, results|
47
- domains.concat(results[:domains])
48
- domains
49
- }
50
- end
51
-
52
- def domain(domain_name)
53
- Domain.new(@sdb, domain_name)
54
- end
55
-
56
- def domain_exists?(domain_name)
57
- domains.include?(domain_name)
58
- end
59
-
60
- def create_domain(domain_name)
61
- @sdb.create_domain(domain_name)
62
- domain(domain_name)
63
- end
64
-
65
- def delete_domain(domain_name)
66
- @sdb.delete_domain(domain_name)
67
- end
68
-
69
- def make_dump(domain, filename)
70
- Dump.new(domain, filename, @logger)
71
- end
72
-
73
- def make_load(domain, filename)
74
- Load.new(domain, filename, @logger)
75
- end
76
-
77
- private
78
- end
79
-
80
- class Domain
81
- attr_reader :name
82
-
83
- def initialize(sdb, name)
84
- @sdb = sdb
85
- @name = name
86
- @item_names = nil
87
- @count = nil
88
- end
89
-
90
- def [](item_name)
91
- @sdb.get_attributes(name, item_name)[:attributes]
92
- end
93
-
94
- def item_names
95
- return @item_names if @item_names
96
- query = Operation.new(@sdb, :query, @name, nil, nil)
97
- @item_names = query.inject([]) {|names, results|
98
- names.concat(results[:items])
99
- names
100
- }
101
- end
102
-
103
- def count
104
- return @count if @count
105
- op = Operation.new(@sdb, :select, "select count(*) from #{name}")
106
- @count = op.inject(0) {|count, results|
107
- count += results[:items].first["Domain"]["Count"].first.to_i
108
- count
109
- }
110
- end
111
-
112
- def items(item_names)
113
- names = item_names.map{|n| "'#{n}'"}.join(', ')
114
- query = "select * from #{name} where itemName() in (#{names})"
115
- select = Operation.new(@sdb, :select, query)
116
- select.inject({}) {|items, results|
117
- results[:items].each do |item|
118
- item_name = item.keys.first
119
- item_value = item.values.first
120
- items[item_name] = item_value
121
- end
122
- items
123
- }
124
- end
125
-
126
- def put(item_name, attributes)
127
- @sdb.put_attributes(@name, item_name, attributes)
128
- end
129
-
130
- def select(query)
131
- op = Operation.new(@sdb, :select, "select * from #{name} where #{query}")
132
- op.inject([]){|items,results|
133
- batch_items = results[:items].map{|pair|
134
- item = pair.values.first
135
- item.merge!({'itemName()' => pair.keys.first})
136
- item
137
- }
138
- items.concat(batch_items)
139
- }
140
- end
141
-
142
- def delete(item_name)
143
- @sdb.delete_attributes(@name, item_name)
144
- end
145
- end
146
-
147
13
  class Task
148
14
  fattr :callback => lambda{}
149
15
  fattr :chunk_size => 100
@@ -0,0 +1,49 @@
1
+ module SDBTools
2
+ class Database
3
+ attr_reader :sdb
4
+ attr_accessor :logger
5
+
6
+ def initialize(access_key=nil, secret_key=nil, options={})
7
+ @logger = (options[:logger] ||= ::Logger.new($stderr))
8
+ @sdb = MeasuredSdbInterface.new(
9
+ options.delete(:sdb_interface) {
10
+ RightAws::SdbInterface.new(access_key, secret_key, options)
11
+ })
12
+ end
13
+
14
+ def domains
15
+ domains_op = Operation.new(@sdb, :list_domains, nil)
16
+ domains_op.inject([]) {|domains, (results, operation)|
17
+ domains.concat(results[:domains])
18
+ domains
19
+ }
20
+ end
21
+
22
+ def domain(domain_name)
23
+ Domain.new(@sdb, domain_name)
24
+ end
25
+
26
+ def domain_exists?(domain_name)
27
+ domains.include?(domain_name)
28
+ end
29
+
30
+ def create_domain(domain_name)
31
+ @sdb.create_domain(domain_name)
32
+ domain(domain_name)
33
+ end
34
+
35
+ def delete_domain(domain_name)
36
+ @sdb.delete_domain(domain_name)
37
+ end
38
+
39
+ def make_dump(domain, filename)
40
+ Dump.new(domain, filename, @logger)
41
+ end
42
+
43
+ def make_load(domain, filename)
44
+ Load.new(domain, filename, @logger)
45
+ end
46
+
47
+ private
48
+ end
49
+ end
@@ -0,0 +1,74 @@
1
+ module SDBTools
2
+ class Domain
3
+ attr_reader :name
4
+
5
+ def initialize(sdb, name)
6
+ @sdb = sdb
7
+ @name = name
8
+ @item_names = nil
9
+ @count = nil
10
+ end
11
+
12
+ def [](item_name)
13
+ @sdb.get_attributes(name, item_name)[:attributes]
14
+ end
15
+
16
+ def item_names
17
+ return @item_names if @item_names
18
+ query = Operation.new(@sdb, :query, @name, nil, nil)
19
+ @item_names = query.inject([]) {|names, results|
20
+ names.concat(results[:items])
21
+ names
22
+ }
23
+ end
24
+
25
+ def count
26
+ return @count if @count
27
+ @count = selection.count
28
+ end
29
+
30
+ def items(item_names)
31
+ names = item_names.map{|n| "'#{n}'"}.join(', ')
32
+ query = "select * from #{name} where itemName() in (#{names})"
33
+ select = Operation.new(@sdb, :select, query)
34
+ select.inject({}) {|items, results|
35
+ results[:items].each do |item|
36
+ item_name = item.keys.first
37
+ item_value = item.values.first
38
+ items[item_name] = item_value
39
+ end
40
+ items
41
+ }
42
+ end
43
+
44
+ def get(item_name, attribute_name=nil)
45
+ @sdb.get_attributes(@name, item_name, attribute_name)
46
+ end
47
+
48
+ def put(item_name, attributes, options={})
49
+ replace = options[:replace] ? :replace : false
50
+ @sdb.put_attributes(@name, item_name, attributes, replace)
51
+ end
52
+
53
+ def selection(options={})
54
+ Selection.new(@sdb, name, options)
55
+ end
56
+
57
+ # Somewhat deprecated. Use #selection() instead
58
+ def select(query)
59
+ op = Operation.new(@sdb, :select, "select * from #{name} where #{query}")
60
+ op.inject([]){|items,(results, operation)|
61
+ batch_items = results[:items].map{|pair|
62
+ item = pair.values.first
63
+ item.merge!({'itemName()' => pair.keys.first})
64
+ item
65
+ }
66
+ items.concat(batch_items)
67
+ }
68
+ end
69
+
70
+ def delete(item_name, attributes={})
71
+ @sdb.delete_attributes(@name, item_name, attributes)
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,48 @@
1
+ require 'sdbtools/transaction'
2
+ require 'delegate'
3
+
4
+ module SDBTools
5
+ class MeasuredSdbInterface < DelegateClass(RightAws::SdbInterface)
6
+ def create_domain(*args, &block)
7
+ Transaction.open("create_domain") do |t|
8
+ t.measure_aws_call{super(*args, &block)}
9
+ end
10
+ end
11
+
12
+ def delete_domain(*args, &block)
13
+ Transaction.open("delete_domain") do |t|
14
+ t.measure_aws_call{super(*args, &block)}
15
+ end
16
+ end
17
+
18
+ def list_domains(*args, &block)
19
+ Transaction.open("list_domains") do |t|
20
+ t.measure_aws_call{super(*args, &block)}
21
+ end
22
+ end
23
+
24
+ def put_attributes(*args, &block)
25
+ Transaction.open("put_attributes") do |t|
26
+ t.measure_aws_call{super(*args, &block)}
27
+ end
28
+ end
29
+
30
+ def get_attributes(*args, &block)
31
+ Transaction.open("get_attributes") do |t|
32
+ t.measure_aws_call{super(*args, &block)}
33
+ end
34
+ end
35
+
36
+ def select(*args, &block)
37
+ Transaction.open("select #{args.first}") do |t|
38
+ t.measure_aws_call{super(*args, &block)}
39
+ end
40
+ end
41
+
42
+ def query(*args, &block)
43
+ Transaction.open("query") do |t|
44
+ t.measure_aws_call{super(*args, &block)}
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,34 @@
1
+ module SDBTools
2
+ # An Operation represents a SimpleDB operation and handles the details of
3
+ # re-requesting "next tokens" until the operation is complete.
4
+ class Operation
5
+ include Enumerable
6
+
7
+ attr_reader :method
8
+ attr_reader :args
9
+ attr_reader :starting_token
10
+
11
+ def initialize(sdb, method, *args)
12
+ @options = args.last.is_a?(Hash) ? args.pop : {}
13
+ @sdb = sdb
14
+ @method = method
15
+ @args = args
16
+ @starting_token = @options[:starting_token]
17
+ end
18
+
19
+ # Yields once for each result set, until there is no next token.
20
+ def each
21
+ Transaction.open(":#{method} operation") do |t|
22
+ next_token = starting_token
23
+ begin
24
+ args = @args.dup
25
+ args << next_token
26
+ results = @sdb.send(@method, *args)
27
+ yield(results, self)
28
+ next_token = results[:next_token]
29
+ end while next_token
30
+ end
31
+ end
32
+
33
+ end
34
+ end
@@ -0,0 +1,212 @@
1
+ require 'arrayfields'
2
+ require 'logger'
3
+
4
+ module SDBTools
5
+ class Selection
6
+ include Enumerable
7
+
8
+ MAX_BATCH_LIMIT = 250
9
+ DEFAULT_RESULT_LIMIT = 100
10
+
11
+ attr_accessor :domain
12
+ attr_accessor :attributes
13
+ attr_accessor :conditions
14
+ attr_reader :limit
15
+ attr_reader :batch_limit
16
+ attr_accessor :offset
17
+ attr_accessor :order_by
18
+ attr_accessor :order
19
+ attr_accessor :sdb
20
+ attr_accessor :logger
21
+
22
+ # For testing
23
+ attr_writer :starting_token
24
+
25
+ def self.quote_name(name)
26
+ if name.to_s =~ /^[A-Z$_][A-Z0-9$_]*$/i
27
+ name.to_s
28
+ else
29
+ "`" + name.to_s.gsub("`", "``") + "`"
30
+ end
31
+ end
32
+
33
+ def self.quote_value(value)
34
+ '"' + value.to_s.gsub(/"/, '""') + '"'
35
+ end
36
+
37
+ def initialize(sdb, domain, options={})
38
+ @sdb = sdb
39
+ @domain = domain.to_s
40
+ @attributes = options.fetch(:attributes) { :all }
41
+ @conditions = options[:conditions].to_s
42
+ @order = options.fetch(:order) { :ascending }
43
+ @order_by = options.fetch(:order_by) { :none }
44
+ @order_by = @order_by.to_s unless @order_by == :none
45
+ self.limit = options.fetch(:limit) { :none }
46
+ self.batch_limit = options.fetch(:batch_limit) { DEFAULT_RESULT_LIMIT }.to_i
47
+ @offset = options.fetch(:offset) { 0 }.to_i
48
+ @logger = options.fetch(:logger){::Logger.new($stderr)}
49
+ end
50
+
51
+ def to_s(query_limit=limit, offset=0)
52
+ "SELECT #{output_list}" \
53
+ " FROM #{quote_name(domain)}" \
54
+ "#{match_expression}" \
55
+ "#{sort_instructions}" \
56
+ "#{limit_clause(query_limit,offset)}"
57
+ end
58
+
59
+ def count_expression
60
+ "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause_for_count}"
61
+ end
62
+
63
+ def offset_count_expression
64
+ "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions} LIMIT #{offset}"
65
+ end
66
+
67
+ def count
68
+ Transaction.open(count_expression) do |t|
69
+ @count ||= count_operation.inject(0){|count, (results, operation)|
70
+ count += results[:items].first["Domain"]["Count"].first.to_i
71
+ }
72
+ end
73
+ end
74
+
75
+ alias_method :size, :count
76
+ alias_method :length, :count
77
+
78
+ def each
79
+ return if limit == 0
80
+ num_items = 0
81
+ Transaction.open(to_s) do
82
+ select_operation(limit, num_items).each do |results, operation|
83
+ results[:items].each do |item|
84
+ yield(item.keys.first, item.values.first)
85
+ num_items += 1
86
+ return if limit != :none && num_items >= limit
87
+ end
88
+ operation.args[0] = to_s(limit, num_items)
89
+ end
90
+ end
91
+ rescue RightAws::AwsError => error
92
+ if error.message =~ /InvalidQueryExpression/
93
+ raise error, error.message.to_s + " (#{to_s(limit, num_items)})", error.backtrace
94
+ else
95
+ raise
96
+ end
97
+ end
98
+
99
+ def results
100
+ @results ||= inject(Arrayfields.new){|results, (name, value)|
101
+ results[name] = value
102
+ results
103
+ }
104
+ end
105
+
106
+ def count_operation
107
+ Operation.new(sdb, :select, count_expression, :starting_token => starting_token)
108
+ end
109
+
110
+ def offset_count_operation
111
+ Operation.new(sdb, :select, offset_count_expression)
112
+ end
113
+
114
+ def select_operation(query_limit=limit, offset=0)
115
+ Operation.new(sdb, :select, to_s(query_limit, offset), :starting_token => starting_token)
116
+ end
117
+
118
+ def starting_token
119
+ @starting_token ||=
120
+ case offset
121
+ when 0 then nil
122
+ else
123
+ op = offset_count_operation
124
+ count = 0
125
+ op.each do |results, operation|
126
+ count += results[:items].first["Domain"]["Count"].first.to_i
127
+ if count == offset || results[:next_token].nil?
128
+ return results[:next_token]
129
+ end
130
+ end
131
+ raise "Failed to find offset #{offset}"
132
+ end
133
+ end
134
+
135
+ def limit=(new_limit)
136
+ case new_limit
137
+ when :none, Integer then
138
+ @limit = new_limit
139
+ else
140
+ raise ArgumentError, "Limit must be integer or :none"
141
+ end
142
+ end
143
+
144
+ def batch_limit=(new_limit)
145
+ case new_limit
146
+ when (1..MAX_BATCH_LIMIT) then
147
+ @batch_limit = new_limit
148
+ else
149
+ raise RangeError, "Limit must be 1..#{MAX_BATCH_LIMIT}"
150
+ end
151
+ end
152
+
153
+ private
154
+
155
+ def quote_name(name)
156
+ self.class.quote_name(name)
157
+ end
158
+
159
+ def quote_value(value)
160
+ self.class.quote_value(value)
161
+ end
162
+
163
+ def output_list
164
+ case attributes
165
+ when Array then attributes.map{|a| quote_name(a)}.join(", ")
166
+ when :all then "*"
167
+ else raise ScriptError, "Bad attributes: #{attributes.inspect}"
168
+ end
169
+ end
170
+
171
+ def match_expression
172
+ if conditions.empty? then "" else " WHERE #{conditions}" end
173
+ end
174
+
175
+ def limit_clause(query_limit=:none, offset=0)
176
+ case query_limit
177
+ when :none then format_limit_clause(batch_limit)
178
+ else
179
+ remaining_query_limit = query_limit - offset
180
+ this_batch_limit =
181
+ remaining_query_limit < batch_limit ? remaining_query_limit : batch_limit
182
+ format_limit_clause(this_batch_limit)
183
+ end
184
+ end
185
+
186
+ def format_limit_clause(this_batch_limit)
187
+ case this_batch_limit
188
+ when DEFAULT_RESULT_LIMIT then ""
189
+ else " LIMIT #{this_batch_limit}"
190
+ end
191
+ end
192
+
193
+ # There are special rules for the LIMIT clause when executing a count(*)
194
+ # select. Normally it specifies batch size, capped at 250. But for count(*)
195
+ # expressions it specifies the max count, which may be arbitrarily large
196
+ def limit_clause_for_count
197
+ case limit
198
+ when :none then ""
199
+ else " LIMIT #{limit}"
200
+ end
201
+ end
202
+
203
+ def sort_instructions
204
+ case order_by
205
+ when :none then ""
206
+ else
207
+ direction = (order == :ascending ? "ASC" : "DESC")
208
+ " ORDER BY #{quote_name(order_by)} #{direction}"
209
+ end
210
+ end
211
+ end
212
+ end
@@ -0,0 +1,119 @@
1
+ require 'benchmark'
2
+
3
+ module SDBTools
4
+
5
+ # SimpleDB is not remotely transactional. A Transaction in this context is
6
+ # just a way to group together a series of SimpleDB requests for the purpose
7
+ # of benchmarking.
8
+ class Transaction
9
+
10
+ def self.open(description, on_close=self.on_close)
11
+ transaction = self.new(description, on_close)
12
+ transaction_stack.push(transaction)
13
+ yield(transaction)
14
+ ensure
15
+ transaction = transaction_stack.pop
16
+ transaction.close
17
+ end
18
+
19
+ # Set the default on_close action. An on_close action receives a Transaction
20
+ # object which is being closed
21
+ def self.on_close=(action)
22
+ @on_close_action = action
23
+ end
24
+
25
+ # Get the default on_close action
26
+ def self.on_close
27
+ if current
28
+ current.on_close
29
+ else
30
+ @on_close_action ||= lambda{|t|}
31
+ end
32
+ end
33
+
34
+ # Usage:
35
+ # Transaction.on_close = Transaction.log_transaction_close(logger)
36
+ def self.log_transaction_close(logger, cutoff_level=:none)
37
+ pattern = "%s \"%s\" (User %0.6u; System %0.6y; CPU %0.6t; Clock %0.6r; Box %0.6f; Reqs %d; Items %d)"
38
+ lambda do |t|
39
+ if cutoff_level == :none || cutoff_level <= t.nesting_level
40
+ prefix = "*" * (t.nesting_level + 1)
41
+ logger.info(
42
+ t.times.format(
43
+ pattern,
44
+ prefix,
45
+ t.description,
46
+ t.box_usage,
47
+ t.request_count,
48
+ t.item_count))
49
+ end
50
+ end
51
+ end
52
+
53
+ def self.add_stats(box_usage, request_count, item_count, times)
54
+ current and current.add_stats(box_usage, request_count, item_count, times)
55
+ end
56
+
57
+ def self.current
58
+ transaction_stack.last
59
+ end
60
+
61
+ def self.transaction_stack
62
+ Thread.current[:sdbtools_transaction_stack] ||= []
63
+ end
64
+
65
+ attr_reader :description
66
+ attr_reader :box_usage
67
+ attr_reader :request_count
68
+ attr_reader :item_count
69
+ attr_reader :times
70
+ attr_reader :on_close
71
+ attr_reader :nesting_level
72
+
73
+ def initialize(description, on_close_action=self.class.on_close)
74
+ @description = description
75
+ @box_usage = 0.0
76
+ @request_count = 0
77
+ @item_count = 0
78
+ @on_close = on_close_action
79
+ @times = Benchmark::Tms.new
80
+ @nesting_level = self.class.transaction_stack.size
81
+ end
82
+
83
+ def close
84
+ self.class.add_stats(@box_usage, @request_count, @item_count, @times)
85
+ @on_close.call(self)
86
+ self
87
+ end
88
+
89
+ def add_stats(box_usage, request_count, item_count, times=Benchmark::Tms.new)
90
+ @request_count += request_count.to_i
91
+ @box_usage += box_usage.to_f
92
+ @item_count += item_count
93
+ @times += times
94
+ end
95
+
96
+ def add_stats_from_aws_response(response)
97
+ response = response.nil? ? {} : response
98
+ box_usage = response[:box_usage].to_f
99
+ item_count = Array(response[:items]).size + Array(response[:domains]).size
100
+ item_count += 1 if response.key?(:attributes)
101
+ add_stats(box_usage, 1, item_count)
102
+ response
103
+ end
104
+
105
+ def measure_aws_call
106
+ add_stats_from_aws_response(time{yield})
107
+ end
108
+
109
+ # Benchmark the block and add its times to the transaction
110
+ def time
111
+ result = nil
112
+ self.times.add! do
113
+ result = yield
114
+ end
115
+ result
116
+ end
117
+
118
+ end
119
+ end
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+ require 'logger'
3
+ require 'irb'
4
+
5
+ require File.expand_path('../lib/sdbtools', File.dirname(__FILE__))
6
+ include SDBTools
7
+ Transaction.on_close = Transaction.log_transaction_close(::Logger.new($stderr))
8
+ IRB.start
@@ -0,0 +1,39 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ module SDBTools
4
+ describe Domain do
5
+ before :each do
6
+ @sdb = stub("SDB")
7
+ end
8
+
9
+ context "given a name and an SDB interface" do
10
+ before :each do
11
+ @it = Domain.new(@sdb, "foo")
12
+ end
13
+
14
+ it "should be able to get selected item attributes" do
15
+ @sdb.should_receive(:get_attributes).
16
+ with("foo", "bar", "baz")
17
+ @it.get("bar", "baz")
18
+ end
19
+
20
+ it "should be able to delete selected item attributes" do
21
+ @sdb.should_receive(:delete_attributes).
22
+ with("foo", "bar", {"baz" => ["buz"]})
23
+ @it.delete("bar", {"baz" => ["buz"]})
24
+ end
25
+
26
+ it "should be able to write selected item attributes" do
27
+ @sdb.should_receive(:put_attributes).
28
+ with("foo", "bar", {"baz" => ["buz"]}, false)
29
+ @it.put("bar", {"baz" => ["buz"]})
30
+ end
31
+
32
+ it "should be able to overwrite selected item attributes" do
33
+ @sdb.should_receive(:put_attributes).
34
+ with("foo", "bar", {"baz" => ["buz"]}, :replace)
35
+ @it.put("bar", {"baz" => ["buz"]}, :replace => true)
36
+ end
37
+ end
38
+ end
39
+ end
@@ -16,5 +16,15 @@ module SDBTools
16
16
  @it.each do break; end
17
17
  end
18
18
  end
19
+
20
+ it "should pass itself into the given block" do
21
+ op = :unset
22
+ @sdb.stub!(:select).and_return({})
23
+ @it = Operation.new(@sdb, :select, "ARG")
24
+ op = @it.each do |results, operation|
25
+ break operation
26
+ end
27
+ op.should equal(@it)
28
+ end
19
29
  end
20
30
  end
@@ -80,10 +80,7 @@ module SDBTools
80
80
  before :each do
81
81
  @it = Selection.new(@sdb, "DOMAIN",
82
82
  :attributes => ["attr foo", "bar"],
83
- :conditions => [
84
- 'bar == "buz"',
85
- "AND",
86
- ['`attr foo` between ? and ?', 4, '1"2']])
83
+ :conditions => 'bar == "buz" AND `attr foo` between "4" and "1""2"')
87
84
  end
88
85
 
89
86
  it "should format quote and interpolate the conditions" do
@@ -102,7 +99,7 @@ module SDBTools
102
99
  end
103
100
 
104
101
  it "should append a limit clause" do
105
- @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250"
102
+ @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar'"
106
103
  end
107
104
 
108
105
  it "should be able to generate a count expression" do
@@ -135,6 +132,38 @@ module SDBTools
135
132
  end
136
133
  end
137
134
 
135
+ context "with a high limit" do
136
+ before :each do
137
+ @it = Selection.new(@sdb, "DOMAIN",
138
+ :attributes => [:foo, :bar],
139
+ :conditions => ["foo == 'bar'"],
140
+ :limit => 251,
141
+ :batch_limit => 250)
142
+ end
143
+
144
+ it "should append a limit clause" do
145
+ @it.to_s.should be == "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250"
146
+ end
147
+
148
+ it "should be able to generate a count expression" do
149
+ @it.count_expression.should be ==
150
+ "SELECT count(*) FROM DOMAIN WHERE foo == 'bar' LIMIT 251"
151
+ end
152
+
153
+ it "should limit results" do
154
+ @sdb = stub("SDB")
155
+ @sdb.should_receive(:select).
156
+ with("SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 250", nil).
157
+ and_return(select_results(("1".."250"), "TOKEN1"))
158
+ @sdb.should_receive(:select).
159
+ with("SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' LIMIT 1", "TOKEN1").
160
+ and_return(select_results(["251"], nil))
161
+ @it.sdb = @sdb
162
+ @it.results.size.should == 251
163
+ @it.results.keys.should == ("1".."251").to_a
164
+ end
165
+ end
166
+
138
167
  context "ordered by an attribute" do
139
168
  before :each do
140
169
  @it = Selection.new(@sdb, "DOMAIN",
@@ -180,11 +209,11 @@ module SDBTools
180
209
 
181
210
  context "with a limit" do
182
211
  before :each do
183
- @it.limit = 250
212
+ @it.limit = 12
184
213
  end
185
214
 
186
215
  specify { @it.to_s.should be ==
187
- "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 250"
216
+ "SELECT foo, bar FROM DOMAIN WHERE foo == 'bar' ORDER BY foo DESC LIMIT 12"
188
217
  }
189
218
  end
190
219
  end
@@ -192,12 +221,12 @@ module SDBTools
192
221
  # We can't yet support large limits. In order to do so, it will be necessary
193
222
  # to implement limit chunking, where the limit is split across multiple
194
223
  # requests of 250 items and a final request of limit % 250 items.
195
- it "should reject limits > 250" do
224
+ it "should reject batch limits > 250" do
196
225
  lambda do
197
- Selection.new(@sdb, "DOMAIN", :limit => 251)
226
+ Selection.new(@sdb, "DOMAIN", :batch_limit => 251)
198
227
  end.should raise_error
199
228
  lambda do
200
- Selection.new(@sdb, "DOMAIN").limit = 251
229
+ Selection.new(@sdb, "DOMAIN").batch_limit = 251
201
230
  end.should raise_error
202
231
  end
203
232
 
@@ -273,10 +302,10 @@ module SDBTools
273
302
  it "should be able to get selection results" do
274
303
  @sdb = stub("SDB")
275
304
  @sdb.should_receive(:select).
276
- with(@it.to_s, "TOKEN2").
305
+ with(@it.to_s(10,0), "TOKEN2").
277
306
  and_return(select_results(["foo", "bar"], "TOKEN5"))
278
307
  @sdb.should_receive(:select).
279
- with(@it.to_s, "TOKEN5").
308
+ with(@it.to_s(10,2), "TOKEN5").
280
309
  and_return(select_results(["baz", "buz"], nil))
281
310
  @it.sdb = @sdb
282
311
  @it.starting_token = "TOKEN2"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sdbtools
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Avdi Grimm
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2010-01-23 00:00:00 -05:00
12
+ date: 2010-01-24 00:00:00 -05:00
13
13
  default_executable: sdbtool
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -78,7 +78,14 @@ files:
78
78
  - VERSION
79
79
  - bin/sdbtool
80
80
  - lib/sdbtools.rb
81
- - lib/selection.rb
81
+ - lib/sdbtools/database.rb
82
+ - lib/sdbtools/domain.rb
83
+ - lib/sdbtools/measured_sdb_interface.rb
84
+ - lib/sdbtools/operation.rb
85
+ - lib/sdbtools/selection.rb
86
+ - lib/sdbtools/transaction.rb
87
+ - script/console
88
+ - spec/domain_spec.rb
82
89
  - spec/operation_spec.rb
83
90
  - spec/selection_spec.rb
84
91
  - spec/spec_helper.rb
@@ -111,6 +118,7 @@ signing_key:
111
118
  specification_version: 3
112
119
  summary: A high-level OO interface to Amazon SimpleDB
113
120
  test_files:
121
+ - spec/domain_spec.rb
114
122
  - spec/selection_spec.rb
115
123
  - spec/operation_spec.rb
116
124
  - spec/spec_helper.rb
@@ -1,197 +0,0 @@
1
- require 'arrayfields'
2
- require 'logger'
3
-
4
- module SDBTools
5
- class Selection
6
- include Enumerable
7
-
8
- MAX_RESULT_LIMIT = 250
9
- DEFAULT_RESULT_LIMIT = 100
10
-
11
- attr_accessor :domain
12
- attr_accessor :attributes
13
- attr_accessor :conditions
14
- attr_reader :limit
15
- attr_accessor :offset
16
- attr_accessor :order_by
17
- attr_accessor :order
18
- attr_accessor :sdb
19
- attr_accessor :logger
20
-
21
- # For testing
22
- attr_writer :starting_token
23
-
24
- def self.quote_name(name)
25
- if name.to_s =~ /^[A-Z$_][A-Z0-9$_]*$/i
26
- name.to_s
27
- else
28
- "`" + name.to_s.gsub("`", "``") + "`"
29
- end
30
- end
31
-
32
- def self.quote_value(value)
33
- '"' + value.to_s.gsub(/"/, '""') + '"'
34
- end
35
-
36
- def initialize(sdb, domain, options={})
37
- @sdb = sdb
38
- @domain = domain.to_s
39
- @attributes = options.fetch(:attributes) { :all }
40
- @conditions = Array(options[:conditions])
41
- @order = options.fetch(:order) { :ascending }
42
- @order_by = options.fetch(:order_by) { :none }
43
- @order_by = @order_by.to_s unless @order_by == :none
44
- self.limit = options.fetch(:limit) { DEFAULT_RESULT_LIMIT }
45
- @offset = options.fetch(:offset) { 0 }.to_i
46
- @logger = options.fetch(:logger){::Logger.new($stderr)}
47
- end
48
-
49
- def to_s
50
- "SELECT #{output_list} FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause}"
51
- end
52
-
53
- def count_expression
54
- "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions}#{limit_clause_for_count}"
55
- end
56
-
57
- def offset_count_expression
58
- "SELECT count(*) FROM #{quote_name(domain)}#{match_expression}#{sort_instructions} LIMIT #{offset}"
59
- end
60
-
61
- def count
62
- @count ||= count_operation.inject(0){|count, results|
63
- count += results[:items].first["Domain"]["Count"].first.to_i
64
- }
65
- end
66
-
67
- alias_method :size, :count
68
- alias_method :length, :count
69
-
70
- def each
71
- return if limit == 0
72
- num_items = 0
73
- select_operation.each do |results|
74
- results[:items].each do |item|
75
- yield(item.keys.first, item.values.first)
76
- num_items += 1
77
- return if limit != :none && num_items >= limit
78
- end
79
- end
80
- end
81
-
82
- def results
83
- @results ||= inject(Arrayfields.new){|results, (name, value)|
84
- results[name] = value
85
- results
86
- }
87
- end
88
-
89
- def count_operation
90
- Operation.new(sdb, :select, count_expression, :starting_token => starting_token)
91
- end
92
-
93
- def offset_count_operation
94
- Operation.new(sdb, :select, offset_count_expression)
95
- end
96
-
97
- def select_operation
98
- Operation.new(sdb, :select, to_s, :starting_token => starting_token)
99
- end
100
-
101
- def starting_token
102
- @starting_token ||=
103
- case offset
104
- when 0 then nil
105
- else
106
- op = offset_count_operation
107
- count = 0
108
- op.each do |results|
109
- count += results[:items].first["Domain"]["Count"].first.to_i
110
- if count == offset || results[:next_token].nil?
111
- return results[:next_token]
112
- end
113
- end
114
- raise "Failed to find offset #{offset}"
115
- end
116
- end
117
-
118
- def limit=(new_limit)
119
- # We can't yet support large limits. In order to do so, it will be necessary
120
- # to implement limit chunking, where the limit is split across multiple
121
- # requests of 250 items and a final request of limit % 250 items.
122
- case new_limit
123
- when :none, (0..MAX_RESULT_LIMIT) then
124
- @limit = new_limit
125
- else
126
- raise RangeError, "Limit must be 0..250 or :none"
127
- end
128
- end
129
-
130
- private
131
-
132
- def quote_name(name)
133
- self.class.quote_name(name)
134
- end
135
-
136
- def quote_value(value)
137
- self.class.quote_value(value)
138
- end
139
-
140
- def output_list
141
- case attributes
142
- when Array then attributes.map{|a| quote_name(a)}.join(", ")
143
- when :all then "*"
144
- else raise ScriptError, "Bad attributes: #{attributes.inspect}"
145
- end
146
- end
147
-
148
- def match_expression
149
- case conditions.size
150
- when 0 then ""
151
- else " WHERE #{prepared_conditions.join(' ')}"
152
- end
153
- end
154
-
155
- def prepared_conditions
156
- conditions.map { |condition|
157
- case condition
158
- when String then condition
159
- when Array then
160
- values = condition.dup
161
- template = values.shift.to_s
162
- template.gsub(/\?/) {|match|
163
- quote_value(values.shift.to_s)
164
- }
165
- else
166
- raise ScriptError, "Bad condition: #{condition.inspect}"
167
- end
168
- }
169
- end
170
-
171
- def limit_clause
172
- case limit
173
- when :none then " LIMIT #{MAX_RESULT_LIMIT}"
174
- when DEFAULT_RESULT_LIMIT then ""
175
- else
176
- batch_limit = limit < MAX_RESULT_LIMIT ? limit : MAX_RESULT_LIMIT
177
- " LIMIT #{batch_limit}"
178
- end
179
- end
180
-
181
- def limit_clause_for_count
182
- case limit
183
- when :none then ""
184
- else limit_clause
185
- end
186
- end
187
-
188
- def sort_instructions
189
- case order_by
190
- when :none then ""
191
- else
192
- direction = (order == :ascending ? "ASC" : "DESC")
193
- " ORDER BY #{quote_name(order_by)} #{direction}"
194
- end
195
- end
196
- end
197
- end