sdb_dal 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/sdb_dal/crypto.rb +51 -0
- data/lib/sdb_dal/domain_attribute_description.rb +33 -16
- data/lib/sdb_dal/domain_object.rb +44 -29
- data/lib/sdb_dal/index_description.rb +15 -4
- data/lib/sdb_dal/memcache_repository.rb +89 -0
- data/lib/sdb_dal/memory_repository.rb +8 -6
- data/lib/sdb_dal/repository.rb +3 -9
- data/lib/sdb_dal/repository_factory.rb +3 -1
- data/lib/sdb_dal/sdb_formatter.rb +2 -2
- data/lib/sdb_dal/storage.rb +28 -9
- data/lib/sdb_dal/uuid.rb +285 -0
- metadata +5 -2
@@ -0,0 +1,51 @@
|
|
1
|
+
require "openssl"
|
2
|
+
include OpenSSL
|
3
|
+
|
4
|
+
module SdbDal
|
5
|
+
class Crypto
|
6
|
+
include OpenSSL::Cipher
|
7
|
+
def initialize(options={})
|
8
|
+
key_dir=options[:key_dir] if options.has_key?(:key_dir)
|
9
|
+
key_dir ||="/tmp"
|
10
|
+
@cipher = Cipher.new("AES-256-CBC")
|
11
|
+
|
12
|
+
if File.exists?(key_dir+'/.cipher_key')
|
13
|
+
@key=File.open(key_dir+'/.cipher_key').read
|
14
|
+
else
|
15
|
+
@key = @cipher.random_key()
|
16
|
+
File.open(key_dir+"/.cipher_key",'w').write(@key)
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
|
21
|
+
|
22
|
+
if File.exists?(key_dir+'/.cipher_iv')
|
23
|
+
@iv=File.open(key_dir+'/.cipher_iv').read
|
24
|
+
else
|
25
|
+
@iv = @cipher.random_iv()
|
26
|
+
File.open(key_dir+"/.cipher_iv",'w').write(@iv)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def encrypt(text)
|
31
|
+
|
32
|
+
@cipher.encrypt(@key,@iv)
|
33
|
+
@cipher.key=@key
|
34
|
+
@cipher.iv = @iv
|
35
|
+
e = @cipher.update(text)
|
36
|
+
e << @cipher.final()
|
37
|
+
Base64.encode64(e)#.chomp
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
def decrypt(text)
|
42
|
+
x=Base64.decode64(text)
|
43
|
+
@cipher.decrypt(@key,@iv)
|
44
|
+
@cipher.key = @key
|
45
|
+
@cipher.iv = @iv
|
46
|
+
d = @cipher.update(x)
|
47
|
+
d << @cipher.final()
|
48
|
+
return d
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -6,15 +6,21 @@ class DomainAttributeDescription
|
|
6
6
|
attr_accessor :value_type
|
7
7
|
attr_accessor :is_primary_key
|
8
8
|
attr_accessor :is_collection
|
9
|
-
|
9
|
+
attr_accessor :is_encrypted
|
10
|
+
|
11
|
+
def self.crypto
|
12
|
+
@@crypto||=Crypto.new
|
13
|
+
@@crypto
|
14
|
+
end
|
10
15
|
|
11
|
-
def initialize(name,type,options)
|
16
|
+
def initialize(name,type,is_encrypted=false,options={})
|
12
17
|
self.name=name
|
13
18
|
self.value_type=type
|
14
19
|
self.is_collection=(type==:reference_set)
|
15
|
-
self.is_collection ||= ( options.has_key?(:is_collection) and options[:is_collection])
|
16
|
-
|
20
|
+
self.is_collection ||= ( options.has_key?(:is_collection) and options[:is_collection])
|
17
21
|
self.is_primary_key=options.has_key?(:is_primary_key) && options[:is_primary_key]==true
|
22
|
+
self.is_encrypted=is_encrypted
|
23
|
+
self.is_encrypted=options[:is_encrypted] if options.has_key?(:is_encrypted)
|
18
24
|
end
|
19
25
|
def is_clob
|
20
26
|
return self.value_type==:clob
|
@@ -23,29 +29,33 @@ class DomainAttributeDescription
|
|
23
29
|
return format_for_sdb_single( value) unless self.is_collection==true
|
24
30
|
result=[]
|
25
31
|
value.each do |single_value|
|
26
|
-
result<<format_for_sdb_single( single_value)
|
32
|
+
result << format_for_sdb_single( single_value)
|
27
33
|
end
|
28
34
|
result
|
29
35
|
end
|
30
36
|
def format_for_sdb_single(value)
|
31
37
|
return nil if value == nil && self.value_type!=:boolean
|
32
38
|
if self.value_type==:integer
|
33
|
-
|
39
|
+
result= format_integer(value)
|
34
40
|
elsif self.value_type==:reference_set
|
35
|
-
|
41
|
+
result= format_reference_set(value)
|
36
42
|
elsif self.value_type==:date
|
37
|
-
|
43
|
+
result= format_date(value)
|
38
44
|
elsif self.value_type==:boolean
|
39
|
-
|
45
|
+
result= format_boolean(value)
|
40
46
|
elsif self.value_type==:unsigned_integer
|
41
|
-
|
47
|
+
result= format_unsigned_integer(value)
|
42
48
|
elsif self.value_type==:float
|
43
|
-
|
49
|
+
result= format_float(value)
|
44
50
|
|
45
51
|
else
|
46
|
-
|
52
|
+
result= format_string(value.to_s)
|
47
53
|
end
|
48
|
-
|
54
|
+
if is_encrypted
|
55
|
+
result=DomainAttributeDescription.crypto.encrypt(result)
|
56
|
+
end
|
57
|
+
|
58
|
+
result
|
49
59
|
end
|
50
60
|
def parse_from_sdb( value)
|
51
61
|
return parse_from_sdb_single( value) unless self.is_collection==true
|
@@ -53,7 +63,7 @@ class DomainAttributeDescription
|
|
53
63
|
|
54
64
|
result=[]
|
55
65
|
value.each do |single_value|
|
56
|
-
result<<parse_from_sdb_single( single_value)
|
66
|
+
result << parse_from_sdb_single( single_value)
|
57
67
|
end
|
58
68
|
result
|
59
69
|
end
|
@@ -61,7 +71,14 @@ class DomainAttributeDescription
|
|
61
71
|
|
62
72
|
return nil if value==nil
|
63
73
|
|
64
|
-
|
74
|
+
if is_encrypted
|
75
|
+
begin
|
76
|
+
value=DomainAttributeDescription.crypto.decrypt(value)
|
77
|
+
rescue
|
78
|
+
#assume wasn't encrypted originally
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
65
82
|
if self.value_type==:integer
|
66
83
|
return parse_integer(value)
|
67
84
|
elsif self.value_type==:date
|
@@ -80,4 +97,4 @@ class DomainAttributeDescription
|
|
80
97
|
return value
|
81
98
|
end
|
82
99
|
end
|
83
|
-
end
|
100
|
+
end
|
@@ -1,13 +1,14 @@
|
|
1
1
|
require "active_support"
|
2
|
-
require 'uuid'
|
2
|
+
require File.dirname(__FILE__) +'/uuid.rb'
|
3
3
|
require File.dirname(__FILE__) +'/domain_attribute_description.rb'
|
4
4
|
require File.dirname(__FILE__) +"/reference.rb"
|
5
5
|
require File.dirname(__FILE__) +"/index_description.rb"
|
6
6
|
require File.dirname(__FILE__) +"/lazy_loading_text.rb"
|
7
7
|
require File.dirname(__FILE__) +"/repository_factory.rb"
|
8
|
+
require File.dirname(__FILE__) +"/crypto.rb"
|
8
9
|
module SdbDal
|
9
10
|
class DomainObject
|
10
|
-
|
11
|
+
@@is_encrypted=false
|
11
12
|
@@items_to_commit=nil
|
12
13
|
@@transaction_depth=0
|
13
14
|
attr_accessor :sdb_key
|
@@ -52,22 +53,22 @@ module SdbDal
|
|
52
53
|
[]
|
53
54
|
end
|
54
55
|
|
55
|
-
def self.index(
|
56
|
-
|
56
|
+
def self.index(index_name,columns,options={})
|
57
|
+
@index_descriptions||={}
|
58
|
+
return if self.index_descriptions[index_name]
|
57
59
|
$columns_xxx=columns
|
58
60
|
class_eval <<-GETTERDONE
|
59
61
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
@@index_names<<:#{name}
|
62
|
+
unless @index_descriptions.has_key?(:#{index_name})
|
63
|
+
@index_names||=[]
|
64
|
+
@index_names<<:#{index_name}
|
64
65
|
|
65
66
|
def self.index_names
|
66
|
-
|
67
|
+
@index_names
|
67
68
|
end
|
68
69
|
|
69
|
-
attribute_description=SdbDal::DomainAttributeDescription.new(:#{
|
70
|
-
|
70
|
+
attribute_description=SdbDal::DomainAttributeDescription.new(:#{index_name},:string,{})
|
71
|
+
@index_descriptions[:#{index_name}]=SdbDal::IndexDescription.new(:#{index_name},$columns_xxx,self.is_encrypted?)
|
71
72
|
end
|
72
73
|
GETTERDONE
|
73
74
|
|
@@ -94,33 +95,40 @@ module SdbDal
|
|
94
95
|
find_scope=":first"
|
95
96
|
end
|
96
97
|
class_eval <<-GETTERDONE
|
97
|
-
def #{
|
98
|
-
return self.class.calculate_#{
|
98
|
+
def #{index_name}
|
99
|
+
return self.class.calculate_#{index_name}(#{getter_params.join(",")})
|
99
100
|
end
|
100
101
|
|
101
|
-
def self.calculate_#{
|
102
|
-
index_description=index_descriptions[:#{
|
102
|
+
def self.calculate_#{index_name}(#{params.join(",")})
|
103
|
+
index_description=index_descriptions[:#{index_name}]
|
103
104
|
h={}
|
104
105
|
#{finder_code}
|
105
106
|
index_description.format_index_entry(@@attribute_descriptions,h)
|
106
107
|
end
|
107
|
-
|
108
|
-
|
109
|
-
options[:
|
110
|
-
options[:
|
111
|
-
options[:index_value]=self.calculate_#{name}(#{params.join(",")})
|
108
|
+
def self.find_by_#{index_name}(#{params.join(",")},options={})
|
109
|
+
options[:params]={:#{index_name}=>self.calculate_#{index_name}(#{params.join(",")})}
|
110
|
+
options[:index]=:#{index_name}
|
111
|
+
options[:index_value]=self.calculate_#{index_name}(#{params.join(",")})
|
112
112
|
find(#{find_scope},options)
|
113
113
|
end
|
114
114
|
GETTERDONE
|
115
115
|
end
|
116
116
|
|
117
117
|
|
118
|
-
|
118
|
+
@index_descriptions ||={}
|
119
119
|
|
120
120
|
def self.index_descriptions
|
121
|
-
|
121
|
+
@index_descriptions || {}
|
122
122
|
end
|
123
123
|
|
124
|
+
def self.is_encrypted?
|
125
|
+
return @@is_encrypted || false
|
126
|
+
end
|
127
|
+
def self.encrypt_me
|
128
|
+
class_eval <<-GETTERDONE
|
129
|
+
@@is_encrypted=true
|
130
|
+
GETTERDONE
|
131
|
+
end
|
124
132
|
def self.data_attribute(name,type,options={})
|
125
133
|
class_eval <<-GETTERDONE
|
126
134
|
@@attribute_descriptions||=HashWithIndifferentAccess.new
|
@@ -152,7 +160,7 @@ module SdbDal
|
|
152
160
|
GETTERDONE
|
153
161
|
return if self.attribute_descriptions.has_key?(name)
|
154
162
|
|
155
|
-
attribute_description=DomainAttributeDescription.new(name,type,options)
|
163
|
+
attribute_description=DomainAttributeDescription.new(name,type,self.is_encrypted?,options)
|
156
164
|
self.attribute_descriptions[name] = attribute_description
|
157
165
|
@primary_key_attribute_names||=[]
|
158
166
|
if attribute_description.is_primary_key
|
@@ -248,7 +256,7 @@ module SdbDal
|
|
248
256
|
|
249
257
|
class_eval <<-GETTERDONE
|
250
258
|
def #{accesser_attribute_name}
|
251
|
-
#{domain_class}.find(self.#{foreign_key_attribute})
|
259
|
+
#{module_name+domain_class.to_s}.find(self.#{foreign_key_attribute})
|
252
260
|
end
|
253
261
|
|
254
262
|
GETTERDONE
|
@@ -281,10 +289,10 @@ module SdbDal
|
|
281
289
|
|
282
290
|
def #{accesser_attribute_name}
|
283
291
|
#unless @accessor_cache.has_key? :#{accesser_attribute_name}
|
284
|
-
through_results= #{through_class}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject.arrayify(self.primary_key)})
|
292
|
+
through_results= #{module_name+through_class.to_s}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject.arrayify(self.primary_key)})
|
285
293
|
result=[]
|
286
294
|
through_results.each do |through_result|
|
287
|
-
item= #{domain_class}.find(through_result.#{foriegn_key_attribute})
|
295
|
+
item= #{module_name+domain_class.to_s}.find(through_result.#{foriegn_key_attribute})
|
288
296
|
result<< item if item
|
289
297
|
end
|
290
298
|
#{order}
|
@@ -296,7 +304,7 @@ module SdbDal
|
|
296
304
|
|
297
305
|
end
|
298
306
|
def connect_#{domain_class.to_s.downcase}(#{domain_class.to_s.downcase})
|
299
|
-
connector=#{through_class}.new
|
307
|
+
connector=#{module_name+through_class.to_s}.new
|
300
308
|
connector.set_map(#{fkey_array_code},DomainObject.arrayify(#{domain_class.to_s.downcase}.primary_key))
|
301
309
|
connector.set_map(#{reflecting_array_code},DomainObject.arrayify(self.primary_key))
|
302
310
|
connector.save
|
@@ -406,8 +414,9 @@ module SdbDal
|
|
406
414
|
# else
|
407
415
|
class_eval <<-XXDONE
|
408
416
|
def #{accesser_attribute_name}
|
417
|
+
|
409
418
|
if !@accessor_cache.has_key? :#{accesser_attribute_name}
|
410
|
-
result= #{domain_class}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject::arrayify(self.primary_key)}#{order})
|
419
|
+
result= #{module_name+domain_class.to_s}.find(:all,:map=>{:keys=>#{reflecting_array_code},:values=>DomainObject::arrayify(self.primary_key)}#{order})
|
411
420
|
@accessor_cache[:#{accesser_attribute_name}]=result || []
|
412
421
|
end
|
413
422
|
return @accessor_cache[:#{accesser_attribute_name}]
|
@@ -537,7 +546,7 @@ module SdbDal
|
|
537
546
|
end
|
538
547
|
def save(options={})
|
539
548
|
if !self.primary_key
|
540
|
-
self.primary_key= UUID.generate
|
549
|
+
self.primary_key= SdbDal::UUID.generate
|
541
550
|
end
|
542
551
|
|
543
552
|
if @@transaction_depth>0
|
@@ -594,6 +603,7 @@ module SdbDal
|
|
594
603
|
|
595
604
|
return self.class.repository
|
596
605
|
end
|
606
|
+
|
597
607
|
class << self
|
598
608
|
def repository(options={})
|
599
609
|
|
@@ -694,5 +704,10 @@ module SdbDal
|
|
694
704
|
end
|
695
705
|
|
696
706
|
end
|
707
|
+
def self.module_name
|
708
|
+
result=""
|
709
|
+
result=self.name.slice(0, self.name.rindex(":")+1) if self.name.rindex(":")
|
710
|
+
result
|
711
|
+
end
|
697
712
|
end
|
698
713
|
end
|
@@ -4,12 +4,18 @@ class IndexDescription < DomainAttributeDescription
|
|
4
4
|
include SdbFormatter
|
5
5
|
attr_accessor :columns
|
6
6
|
|
7
|
+
|
8
|
+
def self.crypto
|
9
|
+
@@crypto||=Crypto.new
|
10
|
+
@@crypto
|
11
|
+
end
|
7
12
|
|
8
|
-
def initialize(name,columns)
|
13
|
+
def initialize(name,columns,is_encrypted=false)
|
9
14
|
self.name=name
|
10
15
|
self.columns=columns
|
11
16
|
self.value_type =:string
|
12
17
|
self.is_primary_key=false
|
18
|
+
self.is_encrypted=is_encrypted
|
13
19
|
end
|
14
20
|
|
15
21
|
def format_index_entry(attribute_descriptions,attribute_values)
|
@@ -18,12 +24,17 @@ class IndexDescription < DomainAttributeDescription
|
|
18
24
|
if column.respond_to?(:transform)
|
19
25
|
result << column.transform(attribute_values[column.source_column] )
|
20
26
|
else
|
21
|
-
result<<attribute_descriptions[column].format_for_sdb(attribute_values[column]).to_s
|
27
|
+
result << attribute_descriptions[column].format_for_sdb(attribute_values[column]).to_s
|
22
28
|
end
|
23
|
-
result<<"&"
|
29
|
+
result << "&"
|
24
30
|
end
|
31
|
+
|
25
32
|
result
|
26
33
|
end
|
34
|
+
def format_for_sdb(value)
|
35
|
+
#don't encrypt because the individual elements are encrypted
|
36
|
+
return value
|
37
|
+
end
|
27
38
|
|
28
39
|
end
|
29
|
-
end
|
40
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module SdbDal
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) +"/memory_storage.rb"
|
4
|
+
|
5
|
+
class MemcacheRepository
|
6
|
+
attr_accessor :use_cache #this here just so interface matches sdb repo
|
7
|
+
attr_accessor :storage
|
8
|
+
|
9
|
+
def initialize(
|
10
|
+
sdb_domain= nil,
|
11
|
+
clob_bucket= nil,
|
12
|
+
aws_key_id= nil,
|
13
|
+
aws_secret_key= nil,
|
14
|
+
memcache_servers = nil ,
|
15
|
+
a_storage=nil,
|
16
|
+
append_table_to_domain=nil
|
17
|
+
)
|
18
|
+
@storage||=Storage.new(aws_key_id,aws_secret_key,memcache_servers,[],{:memory_only=>(aws_key_id==nil) } )
|
19
|
+
@sdb_domain=sdb_domain
|
20
|
+
@clob_bucket=clob_bucket
|
21
|
+
end
|
22
|
+
def pause
|
23
|
+
end
|
24
|
+
|
25
|
+
def clear_session_cache
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
def clear
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
def save(table_name, primary_key, attributes,index_descriptions)
|
34
|
+
key=make_cache_key(table_name,primary_key);
|
35
|
+
record={}
|
36
|
+
|
37
|
+
attributes.each do |description,value|
|
38
|
+
|
39
|
+
record[description.name]=value
|
40
|
+
end
|
41
|
+
record["metadata%table_name"]=table_name
|
42
|
+
record["metadata%primary_key"]=key
|
43
|
+
@storage.put(@clob_bucket,key,record)
|
44
|
+
end
|
45
|
+
def query_ids(table_name,attribute_descriptions,options)
|
46
|
+
raise " not supported for memcache repo"
|
47
|
+
|
48
|
+
end
|
49
|
+
|
50
|
+
def query(table_name,attribute_descriptions,options)
|
51
|
+
raise " not supported for memcache repo"
|
52
|
+
end
|
53
|
+
def find_one(table_name, primary_key,attribute_descriptions)#, non_clob_attribute_names, clob_attribute_names)
|
54
|
+
|
55
|
+
key=make_cache_key(table_name,primary_key)
|
56
|
+
@storage.get(@clob_bucket,key)
|
57
|
+
|
58
|
+
|
59
|
+
end
|
60
|
+
def get_clob(table_name,primary_key,clob_name)
|
61
|
+
raise " not supported for memcache repo"
|
62
|
+
|
63
|
+
end
|
64
|
+
def destroy(table_name, primary_key)
|
65
|
+
key=make_cache_key(table_name,primary_key);
|
66
|
+
@storage.put(@clob_bucket,key,nil)
|
67
|
+
|
68
|
+
end
|
69
|
+
private
|
70
|
+
def flatten_key(key)
|
71
|
+
if key.is_a?( Array)
|
72
|
+
flattened_key=""
|
73
|
+
key.each do |key_part|
|
74
|
+
flattened_key << CGI.escape(key_part.to_s)+"/"
|
75
|
+
end
|
76
|
+
return flattened_key[0..-2]
|
77
|
+
else
|
78
|
+
return CGI.escape(key.to_s)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
def make_cache_key(table_name,primary_key)
|
82
|
+
|
83
|
+
primary_key=flatten_key(primary_key)
|
84
|
+
return "#{@sdb_domain}/#{table_name}/#{primary_key}"
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
@@ -55,7 +55,9 @@ module SdbDal
|
|
55
55
|
index_value=record[index_name.to_sym]
|
56
56
|
save_into_index(table_name, record, index.name, index_value)
|
57
57
|
end
|
58
|
+
|
58
59
|
end
|
60
|
+
# puts record.to_yaml
|
59
61
|
end
|
60
62
|
def save_into_index(table_name,record,index_name,index_value)
|
61
63
|
@indexes[table_name]||={}
|
@@ -67,10 +69,10 @@ module SdbDal
|
|
67
69
|
return
|
68
70
|
end
|
69
71
|
end
|
70
|
-
index<<record
|
72
|
+
index << record
|
71
73
|
@reverse_indexes[table_name]||={}
|
72
74
|
@reverse_indexes[table_name][record["metadata%primary_key"]]||=[]
|
73
|
-
@reverse_indexes[table_name][record["metadata%primary_key"]]<<index
|
75
|
+
@reverse_indexes[table_name][record["metadata%primary_key"]] << index
|
74
76
|
end
|
75
77
|
def retrieve_from_index(table_name,index_name,index_value)
|
76
78
|
return [] unless @indexes[table_name]
|
@@ -89,7 +91,7 @@ module SdbDal
|
|
89
91
|
objects=query(table_name,attribute_descriptions,options)
|
90
92
|
result=[]
|
91
93
|
objects.each do | o|
|
92
|
-
result<<o[primary_key]
|
94
|
+
result << o[primary_key]
|
93
95
|
end
|
94
96
|
result
|
95
97
|
end
|
@@ -163,7 +165,7 @@ module SdbDal
|
|
163
165
|
end
|
164
166
|
end
|
165
167
|
if ok
|
166
|
-
result<<record
|
168
|
+
result << record
|
167
169
|
end
|
168
170
|
|
169
171
|
end
|
@@ -233,7 +235,7 @@ module SdbDal
|
|
233
235
|
if key.is_a?( Array)
|
234
236
|
flattened_key=""
|
235
237
|
key.each do |key_part|
|
236
|
-
flattened_key<<CGI.escape(key_part.to_s)+"/"
|
238
|
+
flattened_key << CGI.escape(key_part.to_s)+"/"
|
237
239
|
end
|
238
240
|
return flattened_key[0..-2]
|
239
241
|
else
|
@@ -251,4 +253,4 @@ module SdbDal
|
|
251
253
|
|
252
254
|
end
|
253
255
|
|
254
|
-
end
|
256
|
+
end
|
data/lib/sdb_dal/repository.rb
CHANGED
@@ -2,7 +2,6 @@ module SdbDal
|
|
2
2
|
$KCODE = 'u'
|
3
3
|
require "aws_sdb"
|
4
4
|
require "sdb_dal/storage.rb"
|
5
|
-
require 'ya2yaml'
|
6
5
|
require File.dirname(__FILE__) +"/memory_repository.rb"
|
7
6
|
require File.dirname(__FILE__) +"/domain_object_cache_item.rb"
|
8
7
|
|
@@ -36,7 +35,6 @@ module SdbDal
|
|
36
35
|
@logger = Logger.new(STDOUT)
|
37
36
|
@logger.level = Logger::ERROR
|
38
37
|
end
|
39
|
-
|
40
38
|
@sdb=AwsSdb::Service.new(:access_key_id=>aws_key_id,:secret_access_key=>aws_secret_key,:url=>"http://sdb.amazonaws.com",:logger=>@logger)
|
41
39
|
@session_cache=MemoryRepository.new
|
42
40
|
|
@@ -131,7 +129,7 @@ module SdbDal
|
|
131
129
|
rescue Exception => e
|
132
130
|
s= "#{e.message}\n#{e.backtrace}"
|
133
131
|
@logger.warn(s) if @logger
|
134
|
-
|
132
|
+
puts e.to_yaml
|
135
133
|
sleep(i*i)
|
136
134
|
end
|
137
135
|
end
|
@@ -201,10 +199,6 @@ module SdbDal
|
|
201
199
|
got_something=true
|
202
200
|
extend_query(query," ['#{key}' >= '#{escape_quotes(attribute_descriptions[key].format_for_sdb_single( value.greater_than_or_equal_to))}']")
|
203
201
|
end
|
204
|
-
if value==false
|
205
|
-
got_something=true
|
206
|
-
extend_query(query," ['#{key}' != 'true' ]")
|
207
|
-
end
|
208
202
|
if !got_something
|
209
203
|
extend_query(query," ['#{key}' = '#{escape_quotes( attribute_descriptions[key].format_for_sdb_single(value))}']")
|
210
204
|
end
|
@@ -366,7 +360,7 @@ module SdbDal
|
|
366
360
|
def sdb_query_with_attributes(table_name,query,max,token=nil)
|
367
361
|
|
368
362
|
@logger.debug( "SDB query:#{table_name}(#{max}) : #{query} #{token}" ) if @logger
|
369
|
-
puts "#{table_name} #{query} (#{max}) #{token}"
|
363
|
+
# puts "#{table_name} #{query} (#{max}) #{token}"
|
370
364
|
20.times do |i|
|
371
365
|
begin
|
372
366
|
return @sdb.query_with_attributes(make_domain_name(table_name),query,max,token)
|
@@ -416,4 +410,4 @@ module SdbDal
|
|
416
410
|
@session_cache.clear
|
417
411
|
end
|
418
412
|
end
|
419
|
-
end
|
413
|
+
end
|
@@ -52,7 +52,7 @@ module SdbFormatter
|
|
52
52
|
end
|
53
53
|
def format_boolean(value)
|
54
54
|
|
55
|
-
return ((value == true) || (value==1))?'true':'false'
|
55
|
+
return ((value == true) || (value==1)) ? 'true' : 'false'
|
56
56
|
end
|
57
57
|
def format_integer(value)
|
58
58
|
|
@@ -116,4 +116,4 @@ module SdbFormatter
|
|
116
116
|
return temp
|
117
117
|
end
|
118
118
|
end
|
119
|
-
end
|
119
|
+
end
|
data/lib/sdb_dal/storage.rb
CHANGED
@@ -13,21 +13,33 @@ class Storage
|
|
13
13
|
@memcache_servers=nil
|
14
14
|
@aws_key_id = nil
|
15
15
|
@aws_secret_key = nil
|
16
|
-
attr_accessor :tokens
|
16
|
+
attr_accessor :tokens
|
17
|
+
attr_accessor :memory_only
|
17
18
|
def initialize( aws_key_id,
|
18
19
|
aws_secret_key,
|
19
20
|
memcache_servers,
|
20
|
-
tokens=[]
|
21
|
+
tokens=[] ,
|
22
|
+
options={})
|
21
23
|
@memcache_servers=memcache_servers
|
22
|
-
|
23
|
-
|
24
|
-
|
24
|
+
if options.has_key?(:memory_only) and options[:memory_only]
|
25
|
+
self.memory_only=true
|
26
|
+
else
|
27
|
+
@aws_key_id=aws_key_id
|
28
|
+
@aws_secret_key=aws_secret_key
|
29
|
+
@tokens=tokens
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
|
25
34
|
if @memcache_servers and @memcache_servers.length>0
|
26
35
|
@cache= MemCache.new @memcache_servers, :namespace => 'my_namespace'
|
27
36
|
end
|
28
37
|
end
|
29
38
|
|
30
39
|
def real_s3
|
40
|
+
if self.memory_only
|
41
|
+
raise "this is a memcache only storage. there is no s3"
|
42
|
+
end
|
31
43
|
unless @conn
|
32
44
|
@conn = S3::AWSAuthConnection.new(@aws_key_id, @aws_secret_key,@tokens,false)
|
33
45
|
end
|
@@ -53,6 +65,9 @@ class Storage
|
|
53
65
|
end
|
54
66
|
def real_s3_get(bucket,key)
|
55
67
|
|
68
|
+
if self.memory_only
|
69
|
+
return nil
|
70
|
+
end
|
56
71
|
20.times do |i|
|
57
72
|
begin
|
58
73
|
response=real_s3.get(bucket,key)
|
@@ -80,6 +95,8 @@ class Storage
|
|
80
95
|
real_s3.create_bucket(bucket,headers)
|
81
96
|
end
|
82
97
|
def real_s3_put(bucket,key,object,attributes)
|
98
|
+
|
99
|
+
return if self.memory_only
|
83
100
|
x=nil
|
84
101
|
20.times do |i|
|
85
102
|
begin
|
@@ -139,10 +156,12 @@ x=nil
|
|
139
156
|
|
140
157
|
end
|
141
158
|
def put(bucket,key,object,attributes={})
|
159
|
+
|
142
160
|
real_s3_put(bucket,key,object,attributes)
|
143
161
|
|
144
|
-
#cache in memcache if not
|
145
|
-
if
|
162
|
+
#cache in memcache if not media file
|
163
|
+
if memory_only ||
|
164
|
+
!attributes ||
|
146
165
|
!attributes.has_key?('Content-Type') ||
|
147
166
|
(attributes['Content-Type'].index('image')!=0 && attributes['Content-Type'].index('audio')!=0 && attributes['Content-Type'].index('video')!=0 )
|
148
167
|
if @cache
|
@@ -170,13 +189,13 @@ x=nil
|
|
170
189
|
end
|
171
190
|
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
172
191
|
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
173
|
-
|
192
|
+
real_s3_query_auth.get(bucket,key)
|
174
193
|
|
175
194
|
|
176
195
|
end
|
177
196
|
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
178
197
|
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
179
|
-
|
198
|
+
real_s3_query_auth.get(bucket,key)
|
180
199
|
|
181
200
|
|
182
201
|
end
|
data/lib/sdb_dal/uuid.rb
ADDED
@@ -0,0 +1,285 @@
|
|
1
|
+
|
2
|
+
#
|
3
|
+
# = uuid.rb - UUID generator
|
4
|
+
#
|
5
|
+
# Author:: Assaf Arkin assaf@labnotes.org
|
6
|
+
# Eric Hodel drbrain@segment7.net
|
7
|
+
# Documentation:: http://trac.labnotes.org/cgi-bin/trac.cgi/wiki/Ruby/UuidGenerator
|
8
|
+
# Copyright:: Copyright (c) 2005-2008 Assaf Arkin, Eric Hodel
|
9
|
+
# License:: MIT and/or Creative Commons Attribution-ShareAlike
|
10
|
+
|
11
|
+
require 'fileutils'
|
12
|
+
require 'thread'
|
13
|
+
require 'tmpdir'
|
14
|
+
|
15
|
+
require 'rubygems'
|
16
|
+
require 'macaddr'
|
17
|
+
|
18
|
+
|
19
|
+
##
|
20
|
+
# = Generating UUIDs
|
21
|
+
#
|
22
|
+
# Call #generate to generate a new UUID. The method returns a string in one of
|
23
|
+
# three formats. The default format is 36 characters long, and contains the 32
|
24
|
+
# hexadecimal octets and hyphens separating the various value parts. The
|
25
|
+
# <tt>:compact</tt> format omits the hyphens, while the <tt>:urn</tt> format
|
26
|
+
# adds the <tt>:urn:uuid</tt> prefix.
|
27
|
+
#
|
28
|
+
# For example:
|
29
|
+
#
|
30
|
+
# uuid = UUID.new
|
31
|
+
#
|
32
|
+
# 10.times do
|
33
|
+
# p uuid.generate
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# = UUIDs in Brief
|
37
|
+
#
|
38
|
+
# UUID (universally unique identifier) are guaranteed to be unique across time
|
39
|
+
# and space.
|
40
|
+
#
|
41
|
+
# A UUID is 128 bit long, and consists of a 60-bit time value, a 16-bit
|
42
|
+
# sequence number and a 48-bit node identifier.
|
43
|
+
#
|
44
|
+
# The time value is taken from the system clock, and is monotonically
|
45
|
+
# incrementing. However, since it is possible to set the system clock
|
46
|
+
# backward, a sequence number is added. The sequence number is incremented
|
47
|
+
# each time the UUID generator is started. The combination guarantees that
|
48
|
+
# identifiers created on the same machine are unique with a high degree of
|
49
|
+
# probability.
|
50
|
+
#
|
51
|
+
# Note that due to the structure of the UUID and the use of sequence number,
|
52
|
+
# there is no guarantee that UUID values themselves are monotonically
|
53
|
+
# incrementing. The UUID value cannot itself be used to sort based on order
|
54
|
+
# of creation.
|
55
|
+
#
|
56
|
+
# To guarantee that UUIDs are unique across all machines in the network,
|
57
|
+
# the IEEE 802 MAC address of the machine's network interface card is used as
|
58
|
+
# the node identifier.
|
59
|
+
#
|
60
|
+
# For more information see {RFC 4122}[http://www.ietf.org/rfc/rfc4122.txt].
|
61
|
+
module SdbDal
|
62
|
+
|
63
|
+
class UUID
|
64
|
+
|
65
|
+
VERSION = '2.0.1'
|
66
|
+
|
67
|
+
##
|
68
|
+
# Clock multiplier. Converts Time (resolution: seconds) to UUID clock
|
69
|
+
# (resolution: 10ns)
|
70
|
+
CLOCK_MULTIPLIER = 10000000
|
71
|
+
|
72
|
+
##
|
73
|
+
# Clock gap is the number of ticks (resolution: 10ns) between two Ruby Time
|
74
|
+
# ticks.
|
75
|
+
CLOCK_GAPS = 100000
|
76
|
+
|
77
|
+
##
|
78
|
+
# Version number stamped into the UUID to identify it as time-based.
|
79
|
+
VERSION_CLOCK = 0x0100
|
80
|
+
|
81
|
+
##
|
82
|
+
# Formats supported by the UUID generator.
|
83
|
+
#
|
84
|
+
# <tt>:default</tt>:: Produces 36 characters, including hyphens separating
|
85
|
+
# the UUID value parts
|
86
|
+
# <tt>:compact</tt>:: Produces a 32 digits (hexadecimal) value with no
|
87
|
+
# hyphens
|
88
|
+
# <tt>:urn</tt>:: Adds the prefix <tt>urn:uuid:</tt> to the default format
|
89
|
+
FORMATS = {
|
90
|
+
:compact => '%08x%04x%04x%04x%012x',
|
91
|
+
:default => '%08x-%04x-%04x-%04x-%012x',
|
92
|
+
:urn => 'urn:uuid:%08x-%04x-%04x-%04x-%012x',
|
93
|
+
}
|
94
|
+
|
95
|
+
##
|
96
|
+
# MAC address (48 bits), sequence number and last clock
|
97
|
+
STATE_FILE_FORMAT = 'SLLQ'
|
98
|
+
|
99
|
+
@state_file = nil
|
100
|
+
@mode = nil
|
101
|
+
@uuid = nil
|
102
|
+
|
103
|
+
##
|
104
|
+
# The access mode of the state file. Set it with state_file.
|
105
|
+
|
106
|
+
def self.mode
|
107
|
+
@mode
|
108
|
+
end
|
109
|
+
|
110
|
+
##
|
111
|
+
# Generates a new UUID string using +format+. See FORMATS for a list of
|
112
|
+
# supported formats.
|
113
|
+
|
114
|
+
def self.generate(format = :default)
|
115
|
+
@uuid ||= new
|
116
|
+
@uuid.generate format
|
117
|
+
end
|
118
|
+
|
119
|
+
##
|
120
|
+
# Creates an empty state file in /var/tmp/ruby-uuid or the windows common
|
121
|
+
# application data directory using mode 0644. Call with a different mode
|
122
|
+
# before creating a UUID generator if you want to open access beyond your
|
123
|
+
# user by default.
|
124
|
+
#
|
125
|
+
# If the default state dir is not writable, UUID falls back to ~/.ruby-uuid.
|
126
|
+
#
|
127
|
+
# State files are not portable across machines.
|
128
|
+
def self.state_file(mode = 0644)
|
129
|
+
return @state_file if @state_file
|
130
|
+
|
131
|
+
@mode = mode
|
132
|
+
|
133
|
+
begin
|
134
|
+
require 'Win32API'
|
135
|
+
|
136
|
+
csidl_common_appdata = 0x0023
|
137
|
+
path = 0.chr * 260
|
138
|
+
get_folder_path = Win32API.new('shell32', 'SHGetFolderPath', 'LLLLP', 'L')
|
139
|
+
get_folder_path.call 0, csidl_common_appdata, 0, 1, path
|
140
|
+
|
141
|
+
state_dir = File.join(path.strip)
|
142
|
+
rescue LoadError
|
143
|
+
state_dir = File.join('', 'var', 'tmp')
|
144
|
+
end
|
145
|
+
|
146
|
+
if File.writable?(state_dir) then
|
147
|
+
@state_file = File.join(state_dir, 'ruby-uuid')
|
148
|
+
else
|
149
|
+
@state_file = File.expand_path(File.join('~', '.ruby-uuid'))
|
150
|
+
end
|
151
|
+
|
152
|
+
@state_file
|
153
|
+
end
|
154
|
+
|
155
|
+
##
|
156
|
+
# Create a new UUID generator. You really only need to do this once.
|
157
|
+
def initialize
|
158
|
+
@drift = 0
|
159
|
+
@last_clock = (Time.now.to_f * CLOCK_MULTIPLIER).to_i
|
160
|
+
@mutex = Mutex.new
|
161
|
+
|
162
|
+
if File.exist?(self.class.state_file) then
|
163
|
+
next_sequence
|
164
|
+
else
|
165
|
+
@mac = Mac.addr.gsub(/:|-/, '').hex & 0x7FFFFFFFFFFF
|
166
|
+
fail "Cannot determine MAC address from any available interface, tried with #{Mac.addr}" if @mac == 0
|
167
|
+
@sequence = rand 0x10000
|
168
|
+
|
169
|
+
open_lock 'w' do |io|
|
170
|
+
write_state io
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
##
|
176
|
+
# Generates a new UUID string using +format+. See FORMATS for a list of
|
177
|
+
# supported formats.
|
178
|
+
def generate(format = :default)
|
179
|
+
template = FORMATS[format]
|
180
|
+
|
181
|
+
raise ArgumentError, "invalid UUID format #{format.inspect}" unless template
|
182
|
+
|
183
|
+
# The clock must be monotonically increasing. The clock resolution is at
|
184
|
+
# best 100 ns (UUID spec), but practically may be lower (on my setup,
|
185
|
+
# around 1ms). If this method is called too fast, we don't have a
|
186
|
+
# monotonically increasing clock, so the solution is to just wait.
|
187
|
+
#
|
188
|
+
# It is possible for the clock to be adjusted backwards, in which case we
|
189
|
+
# would end up blocking for a long time. When backward clock is detected,
|
190
|
+
# we prevent duplicates by asking for a new sequence number and continue
|
191
|
+
# with the new clock.
|
192
|
+
|
193
|
+
clock = @mutex.synchronize do
|
194
|
+
clock = (Time.new.to_f * CLOCK_MULTIPLIER).to_i & 0xFFFFFFFFFFFFFFF0
|
195
|
+
|
196
|
+
if clock > @last_clock then
|
197
|
+
@drift = 0
|
198
|
+
@last_clock = clock
|
199
|
+
elsif clock == @last_clock then
|
200
|
+
drift = @drift += 1
|
201
|
+
|
202
|
+
if drift < 10000 then
|
203
|
+
@last_clock += 1
|
204
|
+
else
|
205
|
+
Thread.pass
|
206
|
+
nil
|
207
|
+
end
|
208
|
+
else
|
209
|
+
next_sequence
|
210
|
+
@last_clock = clock
|
211
|
+
end
|
212
|
+
end until clock
|
213
|
+
|
214
|
+
template % [
|
215
|
+
clock & 0xFFFFFFFF,
|
216
|
+
(clock >> 32) & 0xFFFF,
|
217
|
+
((clock >> 48) & 0xFFFF | VERSION_CLOCK),
|
218
|
+
@sequence & 0xFFFF,
|
219
|
+
@mac & 0xFFFFFFFFFFFF
|
220
|
+
]
|
221
|
+
end
|
222
|
+
|
223
|
+
##
|
224
|
+
# Updates the state file with a new sequence number.
|
225
|
+
def next_sequence
|
226
|
+
open_lock 'r+' do |io|
|
227
|
+
@mac, @sequence, @last_clock = read_state(io)
|
228
|
+
|
229
|
+
io.rewind
|
230
|
+
io.truncate 0
|
231
|
+
|
232
|
+
@sequence += 1
|
233
|
+
|
234
|
+
write_state io
|
235
|
+
end
|
236
|
+
rescue Errno::ENOENT
|
237
|
+
open_lock 'w' do |io|
|
238
|
+
write_state io
|
239
|
+
end
|
240
|
+
ensure
|
241
|
+
@last_clock = (Time.now.to_f * CLOCK_MULTIPLIER).to_i
|
242
|
+
@drift = 0
|
243
|
+
end
|
244
|
+
|
245
|
+
def inspect
|
246
|
+
mac = ("%012x" % @mac).scan(/[0-9a-f]{2}/).join(':')
|
247
|
+
"MAC: #{mac} Sequence: #{@sequence}"
|
248
|
+
end
|
249
|
+
|
250
|
+
protected
|
251
|
+
|
252
|
+
##
|
253
|
+
# Open the state file with an exclusive lock and access mode +mode+.
|
254
|
+
def open_lock(mode)
|
255
|
+
File.open self.class.state_file, mode, self.class.mode do |io|
|
256
|
+
begin
|
257
|
+
io.flock File::LOCK_EX
|
258
|
+
yield io
|
259
|
+
ensure
|
260
|
+
io.flock File::LOCK_UN
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
265
|
+
##
|
266
|
+
# Read the state from +io+
|
267
|
+
def read_state(io)
|
268
|
+
mac1, mac2, seq, last_clock = io.read(32).unpack(STATE_FILE_FORMAT)
|
269
|
+
mac = (mac1 << 32) + mac2
|
270
|
+
|
271
|
+
return mac, seq, last_clock
|
272
|
+
end
|
273
|
+
|
274
|
+
|
275
|
+
##
|
276
|
+
# Write that state to +io+
|
277
|
+
def write_state(io)
|
278
|
+
mac2 = @mac & 0xffffffff
|
279
|
+
mac1 = (@mac >> 32) & 0xffff
|
280
|
+
|
281
|
+
io.write [mac1, mac2, @sequence, @last_clock].pack(STATE_FILE_FORMAT)
|
282
|
+
end
|
283
|
+
|
284
|
+
end
|
285
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sdb_dal
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- David Knight
|
@@ -9,7 +9,7 @@ autorequire: sdb_dal
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2009-01-
|
12
|
+
date: 2009-01-11 00:00:00 -08:00
|
13
13
|
default_executable:
|
14
14
|
dependencies: []
|
15
15
|
|
@@ -26,6 +26,7 @@ files:
|
|
26
26
|
- ./lib/sdb_dal/acts_as_sdb_application.rb
|
27
27
|
- ./lib/sdb_dal/and_condition.rb
|
28
28
|
- ./lib/sdb_dal/attribute_range.rb
|
29
|
+
- ./lib/sdb_dal/crypto.rb
|
29
30
|
- ./lib/sdb_dal/domain_attribute_description.rb
|
30
31
|
- ./lib/sdb_dal/domain_object.rb
|
31
32
|
- ./lib/sdb_dal/domain_object_cache_item.rb
|
@@ -34,6 +35,7 @@ files:
|
|
34
35
|
- ./lib/sdb_dal/index_description.rb
|
35
36
|
- ./lib/sdb_dal/is_null_transform.rb
|
36
37
|
- ./lib/sdb_dal/lazy_loading_text.rb
|
38
|
+
- ./lib/sdb_dal/memcache_repository.rb
|
37
39
|
- ./lib/sdb_dal/memory_repository.rb
|
38
40
|
- ./lib/sdb_dal/memory_storage.rb
|
39
41
|
- ./lib/sdb_dal/or_condition.rb
|
@@ -48,6 +50,7 @@ files:
|
|
48
50
|
- ./lib/sdb_dal/storage.rb
|
49
51
|
- ./lib/sdb_dal/tag_cloud.rb
|
50
52
|
- ./lib/sdb_dal/tracker_description.rb
|
53
|
+
- ./lib/sdb_dal/uuid.rb
|
51
54
|
has_rdoc: false
|
52
55
|
homepage: http://www.yahoo.com/
|
53
56
|
post_install_message:
|