sdb_dal 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/sdb_dal/domain_object.rb +475 -422
- data/lib/sdb_dal/lazy_loading_text.rb +2 -0
- data/lib/sdb_dal/memory_repository.rb +54 -23
- data/lib/sdb_dal/repository.rb +356 -305
- data/lib/sdb_dal/repository_factory.rb +4 -3
- data/lib/sdb_dal/s3.rb +69 -57
- data/lib/sdb_dal/sdb_monkey_patch.rb +47 -0
- data/lib/sdb_dal/storage.rb +8 -4
- metadata +4 -3
@@ -21,14 +21,15 @@ module SdbDal
|
|
21
21
|
else
|
22
22
|
repo_class=SdbDal::Repository
|
23
23
|
end
|
24
|
-
domain_prefix= sdb_dal_config["sdb_domain_prefix"]
|
24
|
+
domain_prefix= sdb_dal_config["sdb_domain_prefix"] || ""
|
25
25
|
clob_bucket= sdb_dal_config["s3_clob_bucket"]
|
26
26
|
aws_key_id = sdb_dal_config["aws_key_id"]
|
27
27
|
aws_secret_key = sdb_dal_config["aws_secret_key"]
|
28
28
|
memcache_servers= sdb_dal_config['memcache_servers']
|
29
29
|
memcache_servers = memcache_servers.split(",") if memcache_servers
|
30
|
-
|
31
|
-
|
30
|
+
|
31
|
+
append_table_to_domain=sdb_dal_config['append_table_to_domain']
|
32
|
+
@repository= repo_class.new(domain_prefix,clob_bucket,aws_key_id,aws_secret_key,memcache_servers,nil,append_table_to_domain)
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
data/lib/sdb_dal/s3.rb
CHANGED
@@ -29,7 +29,6 @@ if (RUBY_VERSION < '1.8.3')
|
|
29
29
|
end
|
30
30
|
end
|
31
31
|
module SdbDal
|
32
|
-
|
33
32
|
# this module has two big classes: AWSAuthConnection and
|
34
33
|
# QueryStringAuthGenerator. both use identical apis, but the first actually
|
35
34
|
# performs the operation, while the second simply outputs urls with the
|
@@ -41,7 +40,7 @@ module S3
|
|
41
40
|
METADATA_PREFIX = 'x-amz-meta-'
|
42
41
|
AMAZON_HEADER_PREFIX = 'x-amz-'
|
43
42
|
AMAZON_TOKEN_HEADER_PREFIX = "x-amz-security-token"
|
44
|
-
|
43
|
+
CANONICAL_STRING_WORTHY_PATH_ARGS=['acl','torrent','logging','requestPayment']
|
45
44
|
# builds the canonical string for signing.
|
46
45
|
def S3.canonical_string(method, bucket="", path="", path_args={}, headers={}, expires=nil)
|
47
46
|
interesting_headers = {}
|
@@ -78,25 +77,24 @@ module S3
|
|
78
77
|
buf << "#{value}\n"
|
79
78
|
end
|
80
79
|
end
|
81
|
-
|
80
|
+
|
82
81
|
# build the path using the bucket and key
|
83
82
|
if not bucket.empty?
|
84
83
|
buf << "/#{bucket}"
|
85
84
|
end
|
86
|
-
# append the key (it might be empty string)
|
85
|
+
# append the key (it might be empty string)
|
87
86
|
# append a slash regardless
|
88
87
|
buf << "/#{path}"
|
89
88
|
|
90
|
-
# if there is an acl, logging, or
|
89
|
+
# if there is an acl, logging, torrent, or requestPayment parameter
|
91
90
|
# add them to the string
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
91
|
+
CANONICAL_STRING_WORTHY_PATH_ARGS.each do |arg_name|
|
92
|
+
if path_args.has_key?(arg_name)
|
93
|
+
buf << "?#{arg_name}"
|
94
|
+
break
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
100
98
|
return buf
|
101
99
|
end
|
102
100
|
|
@@ -121,7 +119,7 @@ module S3
|
|
121
119
|
def S3.path_args_hash_to_string(path_args={})
|
122
120
|
arg_string = ''
|
123
121
|
path_args.each { |k, v|
|
124
|
-
arg_string << k
|
122
|
+
arg_string << k
|
125
123
|
if not v.nil?
|
126
124
|
arg_string << "=#{CGI::escape(v)}"
|
127
125
|
end
|
@@ -138,14 +136,14 @@ module S3
|
|
138
136
|
# some connection pooling.
|
139
137
|
class AWSAuthConnection
|
140
138
|
attr_accessor :calling_format
|
141
|
-
|
142
|
-
def initialize(aws_access_key_id,
|
143
|
-
aws_secret_access_key,
|
139
|
+
|
140
|
+
def initialize(aws_access_key_id,
|
141
|
+
aws_secret_access_key,
|
144
142
|
tokens=Array.new,
|
145
|
-
is_secure=true,
|
146
|
-
server=DEFAULT_HOST,
|
143
|
+
is_secure=true,
|
144
|
+
server=DEFAULT_HOST,
|
147
145
|
port=PORTS_BY_SECURITY[is_secure],
|
148
|
-
calling_format=CallingFormat::
|
146
|
+
calling_format=CallingFormat::REGULAR)
|
149
147
|
@aws_access_key_id = aws_access_key_id
|
150
148
|
@aws_secret_access_key = aws_secret_access_key
|
151
149
|
@server = server
|
@@ -183,26 +181,6 @@ module S3
|
|
183
181
|
make_request('PUT', bucket, CGI::escape(key), {}, headers, object.data, object.metadata)
|
184
182
|
)
|
185
183
|
end
|
186
|
-
def copy(source_bucket, source_key, destination_bucket,destination_key, headers={})
|
187
|
-
headers['x-amz-copy-source']="#{source_bucket}/#{source_key}"
|
188
|
-
headers['x-amz-metadata-directive']="REPLACE "
|
189
|
-
return GetResponse.new(make_request('PUT', destination_bucket, CGI::escape(destination_key),{}, headers))
|
190
|
-
end
|
191
|
-
|
192
|
-
def get_head(bucket, key, headers={})
|
193
|
-
return GetResponse.new(make_request('HEAD',bucket, CGI::escape(key),{}, headers))
|
194
|
-
end
|
195
|
-
def get_content_type(bucket, key, headers={})
|
196
|
-
response= get_head(bucket, key, headers)
|
197
|
-
if response.http_response.code=='404'
|
198
|
-
return nil
|
199
|
-
|
200
|
-
elsif response.http_response.code=='200'
|
201
|
-
return response.http_response.header.content_type
|
202
|
-
end
|
203
|
-
raise response.http_response.code
|
204
|
-
end
|
205
|
-
|
206
184
|
|
207
185
|
def get(bucket, key, headers={})
|
208
186
|
return GetResponse.new(make_request('GET', bucket, CGI::escape(key), {}, headers))
|
@@ -223,7 +201,7 @@ module S3
|
|
223
201
|
def get_bucket_acl(bucket, headers={})
|
224
202
|
return get_acl(bucket, '', headers)
|
225
203
|
end
|
226
|
-
|
204
|
+
|
227
205
|
# returns an xml document representing the access control list.
|
228
206
|
# this could be parsed into an object.
|
229
207
|
def get_acl(bucket, key, headers={})
|
@@ -242,13 +220,46 @@ module S3
|
|
242
220
|
)
|
243
221
|
end
|
244
222
|
|
223
|
+
|
224
|
+
#payer may be RequestPayer::REQUESTER or RequestPayer::BUCKET_OWNER
|
225
|
+
def put_bucket_request_payment( bucket, payer=RequestPayer::REQUESTER, headers={})
|
226
|
+
|
227
|
+
body = "<RequestPaymentConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Payer>#{payer}</Payer></RequestPaymentConfiguration>"
|
228
|
+
|
229
|
+
result= make_request(
|
230
|
+
'PUT',
|
231
|
+
bucket,
|
232
|
+
'',
|
233
|
+
{'requestPayment' => nil },
|
234
|
+
headers,
|
235
|
+
body,
|
236
|
+
{} )
|
237
|
+
return GetResponse.new(result)
|
238
|
+
|
239
|
+
end
|
240
|
+
|
241
|
+
def get_bucket_request_payment( bucket, headers={})
|
242
|
+
|
243
|
+
result= make_request(
|
244
|
+
'GET',
|
245
|
+
bucket,
|
246
|
+
'',
|
247
|
+
{'requestPayment' => nil },
|
248
|
+
headers,
|
249
|
+
'',
|
250
|
+
{} )
|
251
|
+
return GetResponse.new(result)
|
252
|
+
|
253
|
+
end
|
254
|
+
|
255
|
+
|
245
256
|
def list_all_my_buckets(headers={})
|
246
257
|
return ListAllMyBucketsResponse.new(make_request('GET', '', '', {}, headers))
|
247
258
|
end
|
248
259
|
|
249
260
|
private
|
250
261
|
def make_request(method, bucket='', key='', path_args={}, headers={}, data='', metadata={})
|
251
|
-
|
262
|
+
|
252
263
|
# build the domain based on the calling format
|
253
264
|
server = ''
|
254
265
|
if bucket.empty?
|
@@ -257,9 +268,9 @@ module S3
|
|
257
268
|
# does not make sense for vanity domains
|
258
269
|
server = @server
|
259
270
|
elsif @calling_format == CallingFormat::SUBDOMAIN
|
260
|
-
server = "#{bucket}.#{@server}"
|
271
|
+
server = "#{bucket}.#{@server}"
|
261
272
|
elsif @calling_format == CallingFormat::VANITY
|
262
|
-
server = bucket
|
273
|
+
server = bucket
|
263
274
|
else
|
264
275
|
server = @server
|
265
276
|
end
|
@@ -274,11 +285,11 @@ module S3
|
|
274
285
|
path << "/#{key}"
|
275
286
|
|
276
287
|
# build the path_argument string
|
277
|
-
# add the ? in all cases since
|
288
|
+
# add the ? in all cases since
|
278
289
|
# signature and credentials follow path args
|
279
290
|
path << '?'
|
280
|
-
path << S3.path_args_hash_to_string(path_args)
|
281
|
-
|
291
|
+
path << S3.path_args_hash_to_string(path_args)
|
292
|
+
|
282
293
|
http = Net::HTTP.new(server, @port)
|
283
294
|
http.use_ssl = @is_secure
|
284
295
|
http.start do
|
@@ -295,7 +306,6 @@ module S3
|
|
295
306
|
return http.request(req)
|
296
307
|
end
|
297
308
|
end
|
298
|
-
|
299
309
|
end
|
300
310
|
|
301
311
|
def method_to_request_class(method)
|
@@ -306,8 +316,6 @@ module S3
|
|
306
316
|
return Net::HTTP::Put
|
307
317
|
when 'DELETE'
|
308
318
|
return Net::HTTP::Delete
|
309
|
-
when 'HEAD'
|
310
|
-
return Net::HTTP::Head
|
311
319
|
else
|
312
320
|
raise "Unsupported method #{method}"
|
313
321
|
end
|
@@ -348,8 +356,12 @@ module S3
|
|
348
356
|
end
|
349
357
|
end
|
350
358
|
|
359
|
+
module RequestPayer
|
360
|
+
REQUESTER="Requester"
|
361
|
+
BUCKET_OWNER="BucketOwner"
|
362
|
+
end
|
351
363
|
# class for storing calling format constants
|
352
|
-
module CallingFormat
|
364
|
+
module CallingFormat
|
353
365
|
REGULAR = 0 # http://s3.amazonaws.com/bucket/key
|
354
366
|
SUBDOMAIN = 1 # http://bucket.s3.amazonaws.com/key
|
355
367
|
VANITY = 2 # http://<vanity_domain>/key -- vanity_domain resolves to s3.amazonaws.com
|
@@ -366,7 +378,7 @@ module S3
|
|
366
378
|
else
|
367
379
|
build_url_base << "#{server}:#{port}/#{bucket}"
|
368
380
|
end
|
369
|
-
return build_url_base
|
381
|
+
return build_url_base
|
370
382
|
end
|
371
383
|
end
|
372
384
|
|
@@ -418,7 +430,7 @@ module S3
|
|
418
430
|
@curr_entry.owner = Owner.new
|
419
431
|
elsif name == 'CommonPrefixes'
|
420
432
|
@common_prefix_entry = CommonPrefixEntry.new
|
421
|
-
end
|
433
|
+
end
|
422
434
|
end
|
423
435
|
|
424
436
|
# we have one, add him to the entries list
|
@@ -427,7 +439,7 @@ module S3
|
|
427
439
|
if name == 'Name'
|
428
440
|
@properties.name = @curr_text
|
429
441
|
elsif name == 'Prefix' and @is_echoed_prefix
|
430
|
-
@properties.prefix = @curr_text
|
442
|
+
@properties.prefix = @curr_text
|
431
443
|
@is_echoed_prefix = nil
|
432
444
|
elsif name == 'Marker'
|
433
445
|
@properties.marker = @curr_text
|
@@ -437,7 +449,7 @@ module S3
|
|
437
449
|
@properties.delimiter = @curr_text
|
438
450
|
elsif name == 'IsTruncated'
|
439
451
|
@properties.is_truncated = @curr_text == 'true'
|
440
|
-
elsif name == 'NextMarker'
|
452
|
+
elsif name == 'NextMarker'
|
441
453
|
@properties.next_marker = @curr_text
|
442
454
|
elsif name == 'Contents'
|
443
455
|
@entries << @curr_entry
|
@@ -456,7 +468,7 @@ module S3
|
|
456
468
|
elsif name == 'DisplayName'
|
457
469
|
@curr_entry.owner.display_name = @curr_text
|
458
470
|
elsif name == 'CommonPrefixes'
|
459
|
-
@common_prefixes << @common_prefix_entry
|
471
|
+
@common_prefixes << @common_prefix_entry
|
460
472
|
elsif name == 'Prefix'
|
461
473
|
# this is the common prefix for keys that match up to the delimiter
|
462
474
|
@common_prefix_entry.prefix = @curr_text
|
@@ -591,4 +603,4 @@ module S3
|
|
591
603
|
end
|
592
604
|
end
|
593
605
|
end
|
594
|
-
end
|
606
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'time'
|
3
|
+
require 'cgi'
|
4
|
+
require 'uri'
|
5
|
+
require 'net/http'
|
6
|
+
require 'base64'
|
7
|
+
require 'openssl'
|
8
|
+
require 'rexml/document'
|
9
|
+
require 'rexml/xpath'
|
10
|
+
|
11
|
+
module AwsSdb
|
12
|
+
class Service
|
13
|
+
def query_with_attributes(domain, query, max = nil, token = nil)
|
14
|
+
params = {
|
15
|
+
'Action' => 'QueryWithAttributes',
|
16
|
+
'QueryExpression' => query,
|
17
|
+
'DomainName' => domain.to_s
|
18
|
+
}
|
19
|
+
params['NextToken'] =
|
20
|
+
token unless token.nil? || token.empty?
|
21
|
+
params['MaxNumberOfItems'] =
|
22
|
+
max.to_s unless max.nil? || max.to_i == 0
|
23
|
+
doc = call(:get, params)
|
24
|
+
results = {}
|
25
|
+
REXML::XPath.each(doc, '//Item') do |item|
|
26
|
+
item_attributes={}
|
27
|
+
item_name = REXML::XPath.first(item, './Name/text()').to_s
|
28
|
+
|
29
|
+
|
30
|
+
REXML::XPath.each(item, "./Attribute") do |attr|
|
31
|
+
|
32
|
+
key = REXML::XPath.first(attr, './Name/text()').to_s
|
33
|
+
value = REXML::XPath.first(attr, './Value/text()').to_s
|
34
|
+
item_attributes[key]= value
|
35
|
+
end
|
36
|
+
results[item_name]=item_attributes
|
37
|
+
|
38
|
+
end
|
39
|
+
return results, REXML::XPath.first(doc, '//NextToken/text()').to_s
|
40
|
+
end
|
41
|
+
|
42
|
+
|
43
|
+
|
44
|
+
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
data/lib/sdb_dal/storage.rb
CHANGED
@@ -80,13 +80,13 @@ class Storage
|
|
80
80
|
real_s3.create_bucket(bucket,headers)
|
81
81
|
end
|
82
82
|
def real_s3_put(bucket,key,object,attributes)
|
83
|
-
|
83
|
+
x=nil
|
84
84
|
20.times do |i|
|
85
85
|
begin
|
86
|
-
|
87
|
-
|
88
|
-
return
|
86
|
+
x= real_s3.put(bucket,key,S3::S3Object.new(object),attributes)
|
87
|
+
|
89
88
|
|
89
|
+
break
|
90
90
|
rescue =>e
|
91
91
|
s= "#{e.message}\n#{e.backtrace}"
|
92
92
|
puts(s)
|
@@ -95,6 +95,10 @@ class Storage
|
|
95
95
|
#try again
|
96
96
|
end
|
97
97
|
end
|
98
|
+
if x.http_response.code!="200"
|
99
|
+
|
100
|
+
raise "bucket #{bucket} key #{key} response #{x.http_response.code}"
|
101
|
+
end
|
98
102
|
end
|
99
103
|
def delete(bucket,key)
|
100
104
|
if @cache
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sdb_dal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Knight
|
@@ -9,7 +9,7 @@ autorequire: sdb_dal
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date:
|
12
|
+
date: 2009-01-03 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -43,6 +43,7 @@ files:
|
|
43
43
|
- ./lib/sdb_dal/repository_factory.rb
|
44
44
|
- ./lib/sdb_dal/s3.rb
|
45
45
|
- ./lib/sdb_dal/sdb_formatter.rb
|
46
|
+
- ./lib/sdb_dal/sdb_monkey_patch.rb
|
46
47
|
- ./lib/sdb_dal/starts_with_condition.rb
|
47
48
|
- ./lib/sdb_dal/storage.rb
|
48
49
|
- ./lib/sdb_dal/tag_cloud.rb
|
@@ -68,7 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
68
69
|
version:
|
69
70
|
requirements: []
|
70
71
|
|
71
|
-
rubyforge_project:
|
72
|
+
rubyforge_project: sdb-dal
|
72
73
|
rubygems_version: 1.3.1
|
73
74
|
signing_key:
|
74
75
|
specification_version: 2
|