sdb_dal 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/sdb_dal/crypto.rb +51 -0
- data/lib/sdb_dal/domain_attribute_description.rb +33 -16
- data/lib/sdb_dal/domain_object.rb +44 -29
- data/lib/sdb_dal/index_description.rb +15 -4
- data/lib/sdb_dal/memcache_repository.rb +89 -0
- data/lib/sdb_dal/memory_repository.rb +8 -6
- data/lib/sdb_dal/repository.rb +3 -9
- data/lib/sdb_dal/repository_factory.rb +3 -1
- data/lib/sdb_dal/sdb_formatter.rb +2 -2
- data/lib/sdb_dal/storage.rb +28 -9
- data/lib/sdb_dal/uuid.rb +285 -0
- metadata +5 -2
@@ -0,0 +1,51 @@
|
|
1
|
+
require "openssl"
|
2
|
+
include OpenSSL
|
3
|
+
|
4
|
+
module SdbDal
|
5
|
+
class Crypto
|
6
|
+
include OpenSSL::Cipher
|
7
|
+
def initialize(options={})
|
8
|
+
key_dir=options[:key_dir] if options.has_key?(:key_dir)
|
9
|
+
key_dir ||="/tmp"
|
10
|
+
@cipher = Cipher.new("AES-256-CBC")
|
11
|
+
|
12
|
+
if File.exists?(key_dir+'/.cipher_key')
|
13
|
+
@key=File.open(key_dir+'/.cipher_key').read
|
14
|
+
else
|
15
|
+
@key = @cipher.random_key()
|
16
|
+
File.open(key_dir+"/.cipher_key",'w').write(@key)
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
if File.exists?(key_dir+'/.cipher_iv')
|
23
|
+
@iv=File.open(key_dir+'/.cipher_iv').read
|
24
|
+
else
|
25
|
+
@iv = @cipher.random_iv()
|
26
|
+
File.open(key_dir+"/.cipher_iv",'w').write(@iv)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def encrypt(text)
|
31
|
+
|
32
|
+
@cipher.encrypt(@key,@iv)
|
33
|
+
@cipher.key=@key
|
34
|
+
@cipher.iv = @iv
|
35
|
+
e = @cipher.update(text)
|
36
|
+
e << @cipher.final()
|
37
|
+
Base64.encode64(e)#.chomp
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
def decrypt(text)
|
42
|
+
x=Base64.decode64(text)
|
43
|
+
@cipher.decrypt(@key,@iv)
|
44
|
+
@cipher.key = @key
|
45
|
+
@cipher.iv = @iv
|
46
|
+
d = @cipher.update(x)
|
47
|
+
d << @cipher.final()
|
48
|
+
return d
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -6,15 +6,21 @@ class DomainAttributeDescription
|
|
6
6
|
attr_accessor :value_type
|
7
7
|
attr_accessor :is_primary_key
|
8
8
|
attr_accessor :is_collection
|
9
|
-
|
9
|
+
attr_accessor :is_encrypted
|
10
|
+
|
11
|
+
def self.crypto
|
12
|
+
@@crypto||=Crypto.new
|
13
|
+
@@crypto
|
14
|
+
end
|
10
15
|
|
11
|
-
def initialize(name,type,options)
|
16
|
+
def initialize(name,type,is_encrypted=false,options={})
|
12
17
|
self.name=name
|
13
18
|
self.value_type=type
|
14
19
|
self.is_collection=(type==:reference_set)
|
15
|
-
self.is_collection ||= ( options.has_key?(:is_collection) and options[:is_collection])
|
16
|
-
|
20
|
+
self.is_collection ||= ( options.has_key?(:is_collection) and options[:is_collection])
|
17
21
|
self.is_primary_key=options.has_key?(:is_primary_key) && options[:is_primary_key]==true
|
22
|
+
self.is_encrypted=is_encrypted
|
23
|
+
self.is_encrypted=options[:is_encrypted] if options.has_key?(:is_encrypted)
|
18
24
|
end
|
19
25
|
def is_clob
|
20
26
|
return self.value_type==:clob
|
@@ -23,29 +29,33 @@ class DomainAttributeDescription
|
|
23
29
|
return format_for_sdb_single( value) unless self.is_collection==true
|
24
30
|
result=[]
|
25
31
|
value.each do |single_value|
|
26
|
-
result<<format_for_sdb_single( single_value)
|
32
|
+
result << format_for_sdb_single( single_value)
|
27
33
|
end
|
28
34
|
result
|
29
35
|
end
|
30
36
|
def format_for_sdb_single(value)
|
31
37
|
return nil if value == nil && self.value_type!=:boolean
|
32
38
|
if self.value_type==:integer
|
33
|
-
|
39
|
+
result= format_integer(value)
|
34
40
|
elsif self.value_type==:reference_set
|
35
|
-
|
41
|
+
result= format_reference_set(value)
|
36
42
|
elsif self.value_type==:date
|
37
|
-
|
43
|
+
result= format_date(value)
|
38
44
|
elsif self.value_type==:boolean
|
39
|
-
|
45
|
+
result= format_boolean(value)
|
40
46
|
elsif self.value_type==:unsigned_integer
|
41
|
-
|
47
|
+
result= format_unsigned_integer(value)
|
42
48
|
elsif self.value_type==:float
|
43
|
-
|
49
|
+
result= format_float(value)
|
44
50
|
|
45
51
|
else
|
46
|
-
|
52
|
+
result= format_string(value.to_s)
|
47
53
|
end
|
48
|
-
|
54
|
+
if is_encrypted
|
55
|
+
result=DomainAttributeDescription.crypto.encrypt(result)
|
56
|
+
end
|
57
|
+
|
58
|
+
result
|
49
59
|
end
|
50
60
|
def parse_from_sdb( value)
|
51
61
|
return parse_from_sdb_single( value) unless self.is_collection==true
|
@@ -53,7 +63,7 @@ class DomainAttributeDescription
|
|
53
63
|
|
54
64
|
result=[]
|
55
65
|
value.each do |single_value|
|
56
|
-
result<<parse_from_sdb_single( single_value)
|
66
|
+
result << parse_from_sdb_single( single_value)
|
57
67
|
end
|
58
68
|
result
|
59
69
|
end
|
@@ -61,7 +71,14 @@ class DomainAttributeDescription
|
|
61
71
|
|
62
72
|
return nil if value==nil
|
63
73
|
|
64
|
-
|
74
|
+
if is_encrypted
|
75
|
+
begin
|
76
|
+
value=DomainAttributeDescription.crypto.decrypt(value)
|
77
|
+
rescue
|
78
|
+
#assume wasn't encrypted originally
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
65
82
|
if self.value_type==:integer
|
66
83
|
return parse_integer(value)
|
67
84
|
elsif self.value_type==:date
|
@@ -80,4 +97,4 @@ class DomainAttributeDescription
|
|
80
97
|
return value
|
81
98
|
end
|
82
99
|
end
|
83
|
-
end
|
100
|
+
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
require "active_support"
|
2
|
-
require 'uuid'
|
2
|
+
require File.dirname(__FILE__) +'/uuid.rb'
|
3
3
|
require File.dirname(__FILE__) +'/domain_attribute_description.rb'
|
4
4
|
require File.dirname(__FILE__) +"/reference.rb"
|
5
5
|
require File.dirname(__FILE__) +"/index_description.rb"
|
6
6
|
require File.dirname(__FILE__) +"/lazy_loading_text.rb"
|
7
7
|
require File.dirname(__FILE__) +"/repository_factory.rb"
|
8
|
+
require File.dirname(__FILE__) +"/crypto.rb"
|
8
9
|
module SdbDal
|
9
10
|
class DomainObject
|
10
|
-
|
11
|
+
@@is_encrypted=false
|
11
12
|
@@items_to_commit=nil
|
12
13
|
@@transaction_depth=0
|
13
14
|
attr_accessor :sdb_key
|
@@ -52,22 +53,22 @@ module SdbDal
|
|
52
53
|
[]
|
53
54
|
end
|
54
55
|
|
55
|
-
def self.index(
|
56
|
-
|
56
|
+
def self.index(index_name,columns,options={})
|
57
|
+
@index_descriptions||={}
|
58
|
+
return if self.index_descriptions[index_name]
|
57
59
|
$columns_xxx=columns
|
58
60
|
class_eval <<-GETTERDONE
|
59
61
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
@@index_names<<:#{name}
|
62
|
+
unless @index_descriptions.has_key?(:#{index_name})
|
63
|
+
@index_names||=[]
|
64
|
+
@index_names<<:#{index_name}
|
64
65
|
|
65
66
|
def self.index_names
|
66
|
-
|
67
|
+
@index_names
|
67
68
|
end
|
68
69
|
|
69
|
-
attribute_description=SdbDal::DomainAttributeDescription.new(:#{
|
70
|
-
|
70
|
+
attribute_description=SdbDal::DomainAttributeDescription.new(:#{index_name},:string,{})
|
71
|
+
@index_descriptions[:#{index_name}]=SdbDal::IndexDescription.new(:#{index_name},$columns_xxx,self.is_encrypted?)
|
71
72
|
end
|
72
73
|
GETTERDONE
|
73
74
|
|
@@ -94,33 +95,40 @@ module SdbDal
|
|
94
95
|
find_scope=":first"
|
95
96
|
end
|
96
97
|
class_eval <<-GETTERDONE
|
97
|
-
def #{
|
98
|
-
return self.class.calculate_#{
|
98
|
+
def #{index_name}
|
99
|
+
return self.class.calculate_#{index_name}(#{getter_params.join(",")})
|
99
100
|
end
|
100
101
|
|
101
|
-
def self.calculate_#{
|
102
|
-
index_description=index_descriptions[:#{
|
102
|
+
def self.calculate_#{index_name}(#{params.join(",")})
|
103
|
+
index_description=index_descriptions[:#{index_name}]
|
103
104
|
h={}
|
104
105
|
#{finder_code}
|
105
106
|
index_description.format_index_entry(@@attribute_descriptions,h)
|
106
107
|
end
|
107
|
-
|
108
|
-
|
109
|
-
options[:
|
110
|
-
options[:
|
111
|
-
options[:index_value]=self.calculate_#{name}(#{params.join(",")})
|
108
|
+
def self.find_by_#{index_name}(#{params.join(",")},options={})
|
109
|
+
options[:params]={:#{index_name}=>self.calculate_#{index_name}(#{params.join(",")})}
|
110
|
+
options[:index]=:#{index_name}
|
111
|
+
options[:index_value]=self.calculate_#{index_name}(#{params.join(",")})
|
112
112
|
find(#{find_scope},options)
|
113
113
|
end
|
114
114
|
GETTERDONE
|
115
115
|
end
|
116
116
|
|
117
117
|
|
118
|
-
|
118
|
+
@index_descriptions ||={}
|
119
119
|
|
120
120
|
def self.index_descriptions
|
121
|
-
|
121
|
+
@index_descriptions || {}
|
122
122
|
end
|
123
123
|
|
124
|
+
def self.is_encrypted?
|
125
|
+
return @@is_encrypted || false
|
126
|
+
end
|
127
|
+
def self.encrypt_me
|
128
|
+
class_eval <<-GETTERDONE
|
129
|
+
@@is_encrypted=true
|
130
|
+
GETTERDONE
|
131
|
+
end
|
124
132
|
def self.data_attribute(name,type,options={})
|
125
133
|
class_eval <<-GETTERDONE
|
126
134
|
@@attribute_descriptions||=HashWithIndifferentAccess.new
|
@@ -152,7 +160,7 @@ module SdbDal
|
|
152
160
|
GETTERDONE
|
153
161
|
return if self.attribute_descriptions.has_key?(name)
|
154
162
|
|
155
|
-
attribute_description=DomainAttributeDescription.new(name,type,options)
|
163
|
+
attribute_description=DomainAttributeDescription.new(name,type,self.is_encrypted?,options)
|
156
164
|
self.attribute_descriptions[name] = attribute_description
|
157
165
|
@primary_key_attribute_names||=[]
|
158
166
|
if attribute_description.is_primary_key
|
@@ -248,7 +256,7 @@ module SdbDal
|
|
248
256
|
|
249
257
|
class_eval <<-GETTERDONE
|
250
258
|
def #{accesser_attribute_name}
|
251
|
-
#{domain_class}.find(self.#{foreign_key_attribute})
|
259
|
+
#{module_name+domain_class.to_s}.find(self.#{foreign_key_attribute})
|
252
260
|
end
|
253
261
|
|
254
262
|
GETTERDONE
|
@@ -281,10 +289,10 @@ module SdbDal
|
|
281
289
|
|
282
290
|
def #{accesser_attribute_name}
|
283
291
|
#unless @accessor_cache.has_key? :#{accesser_attribute_name}
|
284
|
-
through_results= #{through_class}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject.arrayify(self.primary_key)})
|
292
|
+
through_results= #{module_name+through_class.to_s}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject.arrayify(self.primary_key)})
|
285
293
|
result=[]
|
286
294
|
through_results.each do |through_result|
|
287
|
-
item= #{domain_class}.find(through_result.#{foriegn_key_attribute})
|
295
|
+
item= #{module_name+domain_class.to_s}.find(through_result.#{foriegn_key_attribute})
|
288
296
|
result<< item if item
|
289
297
|
end
|
290
298
|
#{order}
|
@@ -296,7 +304,7 @@ module SdbDal
|
|
296
304
|
|
297
305
|
end
|
298
306
|
def connect_#{domain_class.to_s.downcase}(#{domain_class.to_s.downcase})
|
299
|
-
connector=#{through_class}.new
|
307
|
+
connector=#{module_name+through_class.to_s}.new
|
300
308
|
connector.set_map(#{fkey_array_code},DomainObject.arrayify(#{domain_class.to_s.downcase}.primary_key))
|
301
309
|
connector.set_map(#{reflecting_array_code},DomainObject.arrayify(self.primary_key))
|
302
310
|
connector.save
|
@@ -406,8 +414,9 @@ module SdbDal
|
|
406
414
|
# else
|
407
415
|
class_eval <<-XXDONE
|
408
416
|
def #{accesser_attribute_name}
|
417
|
+
|
409
418
|
if !@accessor_cache.has_key? :#{accesser_attribute_name}
|
410
|
-
result= #{domain_class}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject::arrayify(self.primary_key)}#{order})
|
419
|
+
result= #{module_name+domain_class.to_s}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject::arrayify(self.primary_key)}#{order})
|
411
420
|
@accessor_cache[:#{accesser_attribute_name}]=result || []
|
412
421
|
end
|
413
422
|
return @accessor_cache[:#{accesser_attribute_name}]
|
@@ -537,7 +546,7 @@ module SdbDal
|
|
537
546
|
end
|
538
547
|
def save(options={})
|
539
548
|
if !self.primary_key
|
540
|
-
self.primary_key= UUID.generate
|
549
|
+
self.primary_key= SdbDal::UUID.generate
|
541
550
|
end
|
542
551
|
|
543
552
|
if @@transaction_depth>0
|
@@ -594,6 +603,7 @@ module SdbDal
|
|
594
603
|
|
595
604
|
return self.class.repository
|
596
605
|
end
|
606
|
+
|
597
607
|
class << self
|
598
608
|
def repository(options={})
|
599
609
|
|
@@ -694,5 +704,10 @@ module SdbDal
|
|
694
704
|
end
|
695
705
|
|
696
706
|
end
|
707
|
+
def self.module_name
|
708
|
+
result=""
|
709
|
+
result=self.name.slice(0, self.name.rindex(":")+1) if self.name.rindex(":")
|
710
|
+
result
|
711
|
+
end
|
697
712
|
end
|
698
713
|
end
|
@@ -4,12 +4,18 @@ class IndexDescription < DomainAttributeDescription
|
|
4
4
|
include SdbFormatter
|
5
5
|
attr_accessor :columns
|
6
6
|
|
7
|
+
|
8
|
+
def self.crypto
|
9
|
+
@@crypto||=Crypto.new
|
10
|
+
@@crypto
|
11
|
+
end
|
7
12
|
|
8
|
-
def initialize(name,columns)
|
13
|
+
def initialize(name,columns,is_encrypted=false)
|
9
14
|
self.name=name
|
10
15
|
self.columns=columns
|
11
16
|
self.value_type =:string
|
12
17
|
self.is_primary_key=false
|
18
|
+
self.is_encrypted=is_encrypted
|
13
19
|
end
|
14
20
|
|
15
21
|
def format_index_entry(attribute_descriptions,attribute_values)
|
@@ -18,12 +24,17 @@ class IndexDescription < DomainAttributeDescription
|
|
18
24
|
if column.respond_to?(:transform)
|
19
25
|
result << column.transform(attribute_values[column.source_column] )
|
20
26
|
else
|
21
|
-
result<<attribute_descriptions[column].format_for_sdb(attribute_values[column]).to_s
|
27
|
+
result << attribute_descriptions[column].format_for_sdb(attribute_values[column]).to_s
|
22
28
|
end
|
23
|
-
result<<"&"
|
29
|
+
result << "&"
|
24
30
|
end
|
31
|
+
|
25
32
|
result
|
26
33
|
end
|
34
|
+
def format_for_sdb(value)
|
35
|
+
#don't encrypt because the individual elements are encrypted
|
36
|
+
return value
|
37
|
+
end
|
27
38
|
|
28
39
|
end
|
29
|
-
end
|
40
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module SdbDal
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) +"/memory_storage.rb"
|
4
|
+
|
5
|
+
class MemcacheRepository
|
6
|
+
attr_accessor :use_cache #this here just so interface matches sdb repo
|
7
|
+
attr_accessor :storage
|
8
|
+
|
9
|
+
def initialize(
|
10
|
+
sdb_domain= nil,
|
11
|
+
clob_bucket= nil,
|
12
|
+
aws_key_id= nil,
|
13
|
+
aws_secret_key= nil,
|
14
|
+
memcache_servers = nil ,
|
15
|
+
a_storage=nil,
|
16
|
+
append_table_to_domain=nil
|
17
|
+
)
|
18
|
+
@storage||=Storage.new(aws_key_id,aws_secret_key,memcache_servers,[],{:memory_only=>(aws_key_id==nil) } )
|
19
|
+
@sdb_domain=sdb_domain
|
20
|
+
@clob_bucket=clob_bucket
|
21
|
+
end
|
22
|
+
def pause
|
23
|
+
end
|
24
|
+
|
25
|
+
def clear_session_cache
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
def clear
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
def save(table_name, primary_key, attributes,index_descriptions)
|
34
|
+
key=make_cache_key(table_name,primary_key);
|
35
|
+
record={}
|
36
|
+
|
37
|
+
attributes.each do |description,value|
|
38
|
+
|
39
|
+
record[description.name]=value
|
40
|
+
end
|
41
|
+
record["metadata%table_name"]=table_name
|
42
|
+
record["metadata%primary_key"]=key
|
43
|
+
@storage.put(@clob_bucket,key,record)
|
44
|
+
end
|
45
|
+
def query_ids(table_name,attribute_descriptions,options)
|
46
|
+
raise " not supported for memcache repo"
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
def query(table_name,attribute_descriptions,options)
|
51
|
+
raise " not supported for memcache repo"
|
52
|
+
end
|
53
|
+
def find_one(table_name, primary_key,attribute_descriptions)#, non_clob_attribute_names, clob_attribute_names)
|
54
|
+
|
55
|
+
key=make_cache_key(table_name,primary_key)
|
56
|
+
@storage.get(@clob_bucket,key)
|
57
|
+
|
58
|
+
|
59
|
+
end
|
60
|
+
def get_clob(table_name,primary_key,clob_name)
|
61
|
+
raise " not supported for memcache repo"
|
62
|
+
|
63
|
+
end
|
64
|
+
def destroy(table_name, primary_key)
|
65
|
+
key=make_cache_key(table_name,primary_key);
|
66
|
+
@storage.put(@clob_bucket,key,nil)
|
67
|
+
|
68
|
+
end
|
69
|
+
private
|
70
|
+
def flatten_key(key)
|
71
|
+
if key.is_a?( Array)
|
72
|
+
flattened_key=""
|
73
|
+
key.each do |key_part|
|
74
|
+
flattened_key << CGI.escape(key_part.to_s)+"/"
|
75
|
+
end
|
76
|
+
return flattened_key[0..-2]
|
77
|
+
else
|
78
|
+
return CGI.escape(key.to_s)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
def make_cache_key(table_name,primary_key)
|
82
|
+
|
83
|
+
primary_key=flatten_key(primary_key)
|
84
|
+
return "#{@sdb_domain}/#{table_name}/#{primary_key}"
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
@@ -55,7 +55,9 @@ module SdbDal
|
|
55
55
|
index_value=record[index_name.to_sym]
|
56
56
|
save_into_index(table_name, record, index.name, index_value)
|
57
57
|
end
|
58
|
+
|
58
59
|
end
|
60
|
+
# puts record.to_yaml
|
59
61
|
end
|
60
62
|
def save_into_index(table_name,record,index_name,index_value)
|
61
63
|
@indexes[table_name]||={}
|
@@ -67,10 +69,10 @@ module SdbDal
|
|
67
69
|
return
|
68
70
|
end
|
69
71
|
end
|
70
|
-
index<<record
|
72
|
+
index << record
|
71
73
|
@reverse_indexes[table_name]||={}
|
72
74
|
@reverse_indexes[table_name][record["metadata%primary_key"]]||=[]
|
73
|
-
@reverse_indexes[table_name][record["metadata%primary_key"]]<<index
|
75
|
+
@reverse_indexes[table_name][record["metadata%primary_key"]] << index
|
74
76
|
end
|
75
77
|
def retrieve_from_index(table_name,index_name,index_value)
|
76
78
|
return [] unless @indexes[table_name]
|
@@ -89,7 +91,7 @@ module SdbDal
|
|
89
91
|
objects=query(table_name,attribute_descriptions,options)
|
90
92
|
result=[]
|
91
93
|
objects.each do | o|
|
92
|
-
result<<o[primary_key]
|
94
|
+
result << o[primary_key]
|
93
95
|
end
|
94
96
|
result
|
95
97
|
end
|
@@ -163,7 +165,7 @@ module SdbDal
|
|
163
165
|
end
|
164
166
|
end
|
165
167
|
if ok
|
166
|
-
result<<record
|
168
|
+
result << record
|
167
169
|
end
|
168
170
|
|
169
171
|
end
|
@@ -233,7 +235,7 @@ module SdbDal
|
|
233
235
|
if key.is_a?( Array)
|
234
236
|
flattened_key=""
|
235
237
|
key.each do |key_part|
|
236
|
-
flattened_key<<CGI.escape(key_part.to_s)+"/"
|
238
|
+
flattened_key << CGI.escape(key_part.to_s)+"/"
|
237
239
|
end
|
238
240
|
return flattened_key[0..-2]
|
239
241
|
else
|
@@ -251,4 +253,4 @@ module SdbDal
|
|
251
253
|
|
252
254
|
end
|
253
255
|
|
254
|
-
end
|
256
|
+
end
|
data/lib/sdb_dal/repository.rb
CHANGED
@@ -2,7 +2,6 @@ module SdbDal
|
|
2
2
|
$KCODE = 'u'
|
3
3
|
require "aws_sdb"
|
4
4
|
require "sdb_dal/storage.rb"
|
5
|
-
require 'ya2yaml'
|
6
5
|
require File.dirname(__FILE__) +"/memory_repository.rb"
|
7
6
|
require File.dirname(__FILE__) +"/domain_object_cache_item.rb"
|
8
7
|
|
@@ -36,7 +35,6 @@ module SdbDal
|
|
36
35
|
@logger = Logger.new(STDOUT)
|
37
36
|
@logger.level = Logger::ERROR
|
38
37
|
end
|
39
|
-
|
40
38
|
@sdb=AwsSdb::Service.new(:access_key_id=>aws_key_id,:secret_access_key=>aws_secret_key,:url=>"http://sdb.amazonaws.com",:logger=>@logger)
|
41
39
|
@session_cache=MemoryRepository.new
|
42
40
|
|
@@ -131,7 +129,7 @@ module SdbDal
|
|
131
129
|
rescue Exception => e
|
132
130
|
s= "#{e.message}\n#{e.backtrace}"
|
133
131
|
@logger.warn(s) if @logger
|
134
|
-
|
132
|
+
puts e.to_yaml
|
135
133
|
sleep(i*i)
|
136
134
|
end
|
137
135
|
end
|
@@ -201,10 +199,6 @@ module SdbDal
|
|
201
199
|
got_something=true
|
202
200
|
extend_query(query," ['#{key}' >= '#{escape_quotes(attribute_descriptions[key].format_for_sdb_single( value.greater_than_or_equal_to))}']")
|
203
201
|
end
|
204
|
-
if value==false
|
205
|
-
got_something=true
|
206
|
-
extend_query(query," ['#{key}' != 'true' ]")
|
207
|
-
end
|
208
202
|
if !got_something
|
209
203
|
extend_query(query," ['#{key}' = '#{escape_quotes( attribute_descriptions[key].format_for_sdb_single(value))}']")
|
210
204
|
end
|
@@ -366,7 +360,7 @@ module SdbDal
|
|
366
360
|
def sdb_query_with_attributes(table_name,query,max,token=nil)
|
367
361
|
|
368
362
|
@logger.debug( "SDB query:#{table_name}(#{max}) : #{query} #{token}" ) if @logger
|
369
|
-
puts "#{table_name} #{query} (#{max}) #{token}"
|
363
|
+
# puts "#{table_name} #{query} (#{max}) #{token}"
|
370
364
|
20.times do |i|
|
371
365
|
begin
|
372
366
|
return @sdb.query_with_attributes(make_domain_name(table_name),query,max,token)
|
@@ -416,4 +410,4 @@ module SdbDal
|
|
416
410
|
@session_cache.clear
|
417
411
|
end
|
418
412
|
end
|
419
|
-
end
|
413
|
+
end
|
@@ -52,7 +52,7 @@ module SdbFormatter
|
|
52
52
|
end
|
53
53
|
def format_boolean(value)
|
54
54
|
|
55
|
-
return ((value == true) || (value==1))?'true':'false'
|
55
|
+
return ((value == true) || (value==1)) ? 'true' : 'false'
|
56
56
|
end
|
57
57
|
def format_integer(value)
|
58
58
|
|
@@ -116,4 +116,4 @@ module SdbFormatter
|
|
116
116
|
return temp
|
117
117
|
end
|
118
118
|
end
|
119
|
-
end
|
119
|
+
end
|
data/lib/sdb_dal/storage.rb
CHANGED
@@ -13,21 +13,33 @@ class Storage
|
|
13
13
|
@memcache_servers=nil
|
14
14
|
@aws_key_id = nil
|
15
15
|
@aws_secret_key = nil
|
16
|
-
attr_accessor :tokens
|
16
|
+
attr_accessor :tokens
|
17
|
+
attr_accessor :memory_only
|
17
18
|
def initialize( aws_key_id,
|
18
19
|
aws_secret_key,
|
19
20
|
memcache_servers,
|
20
|
-
tokens=[]
|
21
|
+
tokens=[] ,
|
22
|
+
options={})
|
21
23
|
@memcache_servers=memcache_servers
|
22
|
-
|
23
|
-
|
24
|
-
|
24
|
+
if options.has_key?(:memory_only) and options[:memory_only]
|
25
|
+
self.memory_only=true
|
26
|
+
else
|
27
|
+
@aws_key_id=aws_key_id
|
28
|
+
@aws_secret_key=aws_secret_key
|
29
|
+
@tokens=tokens
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
|
25
34
|
if @memcache_servers and @memcache_servers.length>0
|
26
35
|
@cache= MemCache.new @memcache_servers, :namespace => 'my_namespace'
|
27
36
|
end
|
28
37
|
end
|
29
38
|
|
30
39
|
def real_s3
|
40
|
+
if self.memory_only
|
41
|
+
raise "this is a memcache only storage. there is no s3"
|
42
|
+
end
|
31
43
|
unless @conn
|
32
44
|
@conn = S3::AWSAuthConnection.new(@aws_key_id, @aws_secret_key,@tokens,false)
|
33
45
|
end
|
@@ -53,6 +65,9 @@ class Storage
|
|
53
65
|
end
|
54
66
|
def real_s3_get(bucket,key)
|
55
67
|
|
68
|
+
if self.memory_only
|
69
|
+
return nil
|
70
|
+
end
|
56
71
|
20.times do |i|
|
57
72
|
begin
|
58
73
|
response=real_s3.get(bucket,key)
|
@@ -80,6 +95,8 @@ class Storage
|
|
80
95
|
real_s3.create_bucket(bucket,headers)
|
81
96
|
end
|
82
97
|
def real_s3_put(bucket,key,object,attributes)
|
98
|
+
|
99
|
+
return if self.memory_only
|
83
100
|
x=nil
|
84
101
|
20.times do |i|
|
85
102
|
begin
|
@@ -139,10 +156,12 @@ x=nil
|
|
139
156
|
|
140
157
|
end
|
141
158
|
def put(bucket,key,object,attributes={})
|
159
|
+
|
142
160
|
real_s3_put(bucket,key,object,attributes)
|
143
161
|
|
144
|
-
#cache in memcache if not
|
145
|
-
if
|
162
|
+
#cache in memcache if not media file
|
163
|
+
if memory_only ||
|
164
|
+
!attributes ||
|
146
165
|
!attributes.has_key?('Content-Type') ||
|
147
166
|
(attributes['Content-Type'].index('image')!=0 && attributes['Content-Type'].index('audio')!=0 && attributes['Content-Type'].index('video')!=0 )
|
148
167
|
if @cache
|
@@ -170,13 +189,13 @@ x=nil
|
|
170
189
|
end
|
171
190
|
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
172
191
|
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
173
|
-
|
192
|
+
real_s3_query_auth.get(bucket,key)
|
174
193
|
|
175
194
|
|
176
195
|
end
|
177
196
|
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
178
197
|
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
179
|
-
|
198
|
+
real_s3_query_auth.get(bucket,key)
|
180
199
|
|
181
200
|
|
182
201
|
end
|
data/lib/sdb_dal/uuid.rb
ADDED
@@ -0,0 +1,285 @@
|
|
1
|
+
|
2
|
+
#
|
3
|
+
# = uuid.rb - UUID generator
|
4
|
+
#
|
5
|
+
# Author:: Assaf Arkin assaf@labnotes.org
|
6
|
+
# Eric Hodel drbrain@segment7.net
|
7
|
+
# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/Ruby/UuidGenerator
|
8
|
+
# Copyright:: Copyright (c) 2005-2008 Assaf Arkin, Eric Hodel
|
9
|
+
# License:: MIT and/or Creative Commons Attribution-ShareAlike
|
10
|
+
|
11
|
+
require 'fileutils'
|
12
|
+
require 'thread'
|
13
|
+
require 'tmpdir'
|
14
|
+
|
15
|
+
require 'rubygems'
|
16
|
+
require 'macaddr'
|
17
|
+
|
18
|
+
|
19
|
+
##
|
20
|
+
# = Generating UUIDs
|
21
|
+
#
|
22
|
+
# Call #generate to generate a new UUID. The method returns a string in one of
|
23
|
+
# three formats. The default format is 36 characters long, and contains the 32
|
24
|
+
# hexadecimal octets and hyphens separating the various value parts. The
|
25
|
+
# <tt>:compact</tt> format omits the hyphens, while the <tt>:urn</tt> format
|
26
|
+
# adds the <tt>:urn:uuid</tt> prefix.
|
27
|
+
#
|
28
|
+
# For example:
|
29
|
+
#
|
30
|
+
# uuid = UUID.new
|
31
|
+
#
|
32
|
+
# 10.times do
|
33
|
+
# p uuid.generate
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# = UUIDs in Brief
|
37
|
+
#
|
38
|
+
# UUID (universally unique identifier) are guaranteed to be unique across time
|
39
|
+
# and space.
|
40
|
+
#
|
41
|
+
# A UUID is 128 bit long, and consists of a 60-bit time value, a 16-bit
|
42
|
+
# sequence number and a 48-bit node identifier.
|
43
|
+
#
|
44
|
+
# The time value is taken from the system clock, and is monotonically
|
45
|
+
# incrementing. However, since it is possible to set the system clock
|
46
|
+
# backward, a sequence number is added. The sequence number is incremented
|
47
|
+
# each time the UUID generator is started. The combination guarantees that
|
48
|
+
# identifiers created on the same machine are unique with a high degree of
|
49
|
+
# probability.
|
50
|
+
#
|
51
|
+
# Note that due to the structure of the UUID and the use of sequence number,
|
52
|
+
# there is no guarantee that UUID values themselves are monotonically
|
53
|
+
# incrementing. The UUID value cannot itself be used to sort based on order
|
54
|
+
# of creation.
|
55
|
+
#
|
56
|
+
# To guarantee that UUIDs are unique across all machines in the network,
|
57
|
+
# the IEEE 802 MAC address of the machine's network interface card is used as
|
58
|
+
# the node identifier.
|
59
|
+
#
|
60
|
+
# For more information see {RFC 4122}[http://www.ietf.org/rfc/rfc4122.txt].
|
61
|
+
module SdbDal
|
62
|
+
|
63
|
+
class UUID
|
64
|
+
|
65
|
+
VERSION = '2.0.1'
|
66
|
+
|
67
|
+
##
|
68
|
+
# Clock multiplier. Converts Time (resolution: seconds) to UUID clock
|
69
|
+
# (resolution: 10ns)
|
70
|
+
CLOCK_MULTIPLIER = 10000000
|
71
|
+
|
72
|
+
##
|
73
|
+
# Clock gap is the number of ticks (resolution: 10ns) between two Ruby Time
|
74
|
+
# ticks.
|
75
|
+
CLOCK_GAPS = 100000
|
76
|
+
|
77
|
+
##
|
78
|
+
# Version number stamped into the UUID to identify it as time-based.
|
79
|
+
VERSION_CLOCK = 0x0100
|
80
|
+
|
81
|
+
##
|
82
|
+
# Formats supported by the UUID generator.
|
83
|
+
#
|
84
|
+
# <tt>:default</tt>:: Produces 36 characters, including hyphens separating
|
85
|
+
# the UUID value parts
|
86
|
+
# <tt>:compact</tt>:: Produces a 32 digits (hexadecimal) value with no
|
87
|
+
# hyphens
|
88
|
+
# <tt>:urn</tt>:: Adds the prefix <tt>urn:uuid:</tt> to the default format
|
89
|
+
FORMATS = {
|
90
|
+
:compact => '%08x%04x%04x%04x%012x',
|
91
|
+
:default => '%08x-%04x-%04x-%04x-%012x',
|
92
|
+
:urn => 'urn:uuid:%08x-%04x-%04x-%04x-%012x',
|
93
|
+
}
|
94
|
+
|
95
|
+
##
|
96
|
+
# MAC address (48 bits), sequence number and last clock
|
97
|
+
STATE_FILE_FORMAT = 'SLLQ'
|
98
|
+
|
99
|
+
@state_file = nil
|
100
|
+
@mode = nil
|
101
|
+
@uuid = nil
|
102
|
+
|
103
|
+
##
|
104
|
+
# The access mode of the state file. Set it with state_file.
|
105
|
+
|
106
|
+
def self.mode
|
107
|
+
@mode
|
108
|
+
end
|
109
|
+
|
110
|
+
##
|
111
|
+
# Generates a new UUID string using +format+. See FORMATS for a list of
|
112
|
+
# supported formats.
|
113
|
+
|
114
|
+
def self.generate(format = :default)
|
115
|
+
@uuid ||= new
|
116
|
+
@uuid.generate format
|
117
|
+
end
|
118
|
+
|
119
|
+
##
|
120
|
+
# Creates an empty state file in /var/tmp/ruby-uuid or the windows common
|
121
|
+
# application data directory using mode 0644. Call with a different mode
|
122
|
+
# before creating a UUID generator if you want to open access beyond your
|
123
|
+
# user by default.
|
124
|
+
#
|
125
|
+
# If the default state dir is not writable, UUID falls back to ~/.ruby-uuid.
|
126
|
+
#
|
127
|
+
# State files are not portable across machines.
|
128
|
+
def self.state_file(mode = 0644)
|
129
|
+
return @state_file if @state_file
|
130
|
+
|
131
|
+
@mode = mode
|
132
|
+
|
133
|
+
begin
|
134
|
+
require 'Win32API'
|
135
|
+
|
136
|
+
csidl_common_appdata = 0x0023
|
137
|
+
path = 0.chr * 260
|
138
|
+
get_folder_path = Win32API.new('shell32', 'SHGetFolderPath', 'LLLLP', 'L')
|
139
|
+
get_folder_path.call 0, csidl_common_appdata, 0, 1, path
|
140
|
+
|
141
|
+
state_dir = File.join(path.strip)
|
142
|
+
rescue LoadError
|
143
|
+
state_dir = File.join('', 'var', 'tmp')
|
144
|
+
end
|
145
|
+
|
146
|
+
if File.writable?(state_dir) then
|
147
|
+
@state_file = File.join(state_dir, 'ruby-uuid')
|
148
|
+
else
|
149
|
+
@state_file = File.expand_path(File.join('~', '.ruby-uuid'))
|
150
|
+
end
|
151
|
+
|
152
|
+
@state_file
|
153
|
+
end
|
154
|
+
|
155
|
+
##
|
156
|
+
# Create a new UUID generator. You really only need to do this once.
|
157
|
+
def initialize
|
158
|
+
@drift = 0
|
159
|
+
@last_clock = (Time.now.to_f * CLOCK_MULTIPLIER).to_i
|
160
|
+
@mutex = Mutex.new
|
161
|
+
|
162
|
+
if File.exist?(self.class.state_file) then
|
163
|
+
next_sequence
|
164
|
+
else
|
165
|
+
@mac = Mac.addr.gsub(/:|-/, '').hex & 0x7FFFFFFFFFFF
|
166
|
+
fail "Cannot determine MAC address from any available interface, tried with #{Mac.addr}" if @mac == 0
|
167
|
+
@sequence = rand 0x10000
|
168
|
+
|
169
|
+
open_lock 'w' do |io|
|
170
|
+
write_state io
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
##
|
176
|
+
# Generates a new UUID string using +format+. See FORMATS for a list of
|
177
|
+
# supported formats.
|
178
|
+
def generate(format = :default)
|
179
|
+
template = FORMATS[format]
|
180
|
+
|
181
|
+
raise ArgumentError, "invalid UUID format #{format.inspect}" unless template
|
182
|
+
|
183
|
+
# The clock must be monotonically increasing. The clock resolution is at
|
184
|
+
# best 100 ns (UUID spec), but practically may be lower (on my setup,
|
185
|
+
# around 1ms). If this method is called too fast, we don't have a
|
186
|
+
# monotonically increasing clock, so the solution is to just wait.
|
187
|
+
#
|
188
|
+
# It is possible for the clock to be adjusted backwards, in which case we
|
189
|
+
# would end up blocking for a long time. When backward clock is detected,
|
190
|
+
# we prevent duplicates by asking for a new sequence number and continue
|
191
|
+
# with the new clock.
|
192
|
+
|
193
|
+
clock = @mutex.synchronize do
|
194
|
+
clock = (Time.new.to_f * CLOCK_MULTIPLIER).to_i & 0xFFFFFFFFFFFFFFF0
|
195
|
+
|
196
|
+
if clock > @last_clock then
|
197
|
+
@drift = 0
|
198
|
+
@last_clock = clock
|
199
|
+
elsif clock == @last_clock then
|
200
|
+
drift = @drift += 1
|
201
|
+
|
202
|
+
if drift < 10000 then
|
203
|
+
@last_clock += 1
|
204
|
+
else
|
205
|
+
Thread.pass
|
206
|
+
nil
|
207
|
+
end
|
208
|
+
else
|
209
|
+
next_sequence
|
210
|
+
@last_clock = clock
|
211
|
+
end
|
212
|
+
end until clock
|
213
|
+
|
214
|
+
template % [
|
215
|
+
clock & 0xFFFFFFFF,
|
216
|
+
(clock >> 32) & 0xFFFF,
|
217
|
+
((clock >> 48) & 0xFFFF | VERSION_CLOCK),
|
218
|
+
@sequence & 0xFFFF,
|
219
|
+
@mac & 0xFFFFFFFFFFFF
|
220
|
+
]
|
221
|
+
end
|
222
|
+
|
223
|
+
##
|
224
|
+
# Updates the state file with a new sequence number.
|
225
|
+
def next_sequence
|
226
|
+
open_lock 'r+' do |io|
|
227
|
+
@mac, @sequence, @last_clock = read_state(io)
|
228
|
+
|
229
|
+
io.rewind
|
230
|
+
io.truncate 0
|
231
|
+
|
232
|
+
@sequence += 1
|
233
|
+
|
234
|
+
write_state io
|
235
|
+
end
|
236
|
+
rescue Errno::ENOENT
|
237
|
+
open_lock 'w' do |io|
|
238
|
+
write_state io
|
239
|
+
end
|
240
|
+
ensure
|
241
|
+
@last_clock = (Time.now.to_f * CLOCK_MULTIPLIER).to_i
|
242
|
+
@drift = 0
|
243
|
+
end
|
244
|
+
|
245
|
+
def inspect
|
246
|
+
mac = ("%012x" % @mac).scan(/[0-9a-f]{2}/).join(':')
|
247
|
+
"MAC: #{mac} Sequence: #{@sequence}"
|
248
|
+
end
|
249
|
+
|
250
|
+
protected
|
251
|
+
|
252
|
+
##
|
253
|
+
# Open the state file with an exclusive lock and access mode +mode+.
|
254
|
+
def open_lock(mode)
|
255
|
+
File.open self.class.state_file, mode, self.class.mode do |io|
|
256
|
+
begin
|
257
|
+
io.flock File::LOCK_EX
|
258
|
+
yield io
|
259
|
+
ensure
|
260
|
+
io.flock File::LOCK_UN
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
##
|
266
|
+
# Read the state from +io+
|
267
|
+
def read_state(io)
|
268
|
+
mac1, mac2, seq, last_clock = io.read(32).unpack(STATE_FILE_FORMAT)
|
269
|
+
mac = (mac1 << 32) + mac2
|
270
|
+
|
271
|
+
return mac, seq, last_clock
|
272
|
+
end
|
273
|
+
|
274
|
+
|
275
|
+
##
|
276
|
+
# Write that state to +io+
|
277
|
+
def write_state(io)
|
278
|
+
mac2 = @mac & 0xffffffff
|
279
|
+
mac1 = (@mac >> 32) & 0xffff
|
280
|
+
|
281
|
+
io.write [mac1, mac2, @sequence, @last_clock].pack(STATE_FILE_FORMAT)
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
285
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sdb_dal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Knight
|
@@ -9,7 +9,7 @@ autorequire: sdb_dal
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-01-
|
12
|
+
date: 2009-01-11 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -26,6 +26,7 @@ files:
|
|
26
26
|
- ./lib/sdb_dal/acts_as_sdb_application.rb
|
27
27
|
- ./lib/sdb_dal/and_condition.rb
|
28
28
|
- ./lib/sdb_dal/attribute_range.rb
|
29
|
+
- ./lib/sdb_dal/crypto.rb
|
29
30
|
- ./lib/sdb_dal/domain_attribute_description.rb
|
30
31
|
- ./lib/sdb_dal/domain_object.rb
|
31
32
|
- ./lib/sdb_dal/domain_object_cache_item.rb
|
@@ -34,6 +35,7 @@ files:
|
|
34
35
|
- ./lib/sdb_dal/index_description.rb
|
35
36
|
- ./lib/sdb_dal/is_null_transform.rb
|
36
37
|
- ./lib/sdb_dal/lazy_loading_text.rb
|
38
|
+
- ./lib/sdb_dal/memcache_repository.rb
|
37
39
|
- ./lib/sdb_dal/memory_repository.rb
|
38
40
|
- ./lib/sdb_dal/memory_storage.rb
|
39
41
|
- ./lib/sdb_dal/or_condition.rb
|
@@ -48,6 +50,7 @@ files:
|
|
48
50
|
- ./lib/sdb_dal/storage.rb
|
49
51
|
- ./lib/sdb_dal/tag_cloud.rb
|
50
52
|
- ./lib/sdb_dal/tracker_description.rb
|
53
|
+
- ./lib/sdb_dal/uuid.rb
|
51
54
|
has_rdoc: false
|
52
55
|
homepage: http://www.yahoo.com/
|
53
56
|
post_install_message:
|