sdb_dal 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/sdb_dal/and_condition.rb +10 -0
- data/lib/sdb_dal/attribute_range.rb +57 -0
- data/lib/sdb_dal/domain_attribute_description.rb +83 -0
- data/lib/sdb_dal/domain_object.rb +642 -0
- data/lib/sdb_dal/domain_object_cache_item.rb +14 -0
- data/lib/sdb_dal/equals_condition.rb +18 -0
- data/lib/sdb_dal/geo.rb +188 -0
- data/lib/sdb_dal/index_description.rb +29 -0
- data/lib/sdb_dal/is_null_transform.rb +16 -0
- data/lib/sdb_dal/lazy_loading_text.rb +27 -0
- data/lib/sdb_dal/memory_repository.rb +168 -0
- data/lib/sdb_dal/memory_storage.rb +29 -0
- data/lib/sdb_dal/or_condition.rb +53 -0
- data/lib/sdb_dal/query_string_auth_generator.rb +166 -0
- data/lib/sdb_dal/reference.rb +28 -0
- data/lib/sdb_dal/repository.rb +366 -0
- data/lib/sdb_dal/s3.rb +594 -0
- data/lib/sdb_dal/sdb_formatter.rb +119 -0
- data/lib/sdb_dal/starts_with_condition.rb +21 -0
- data/lib/sdb_dal/storage.rb +188 -0
- data/lib/sdb_dal/tag_cloud.rb +56 -0
- data/lib/sdb_dal/tracker_description.rb +26 -0
- data/lib/sdb_dal.rb +8 -0
- metadata +75 -0
@@ -0,0 +1,21 @@
|
|
1
|
+
module SdbDal
|
2
|
+
|
3
|
+
|
4
|
+
class StartsWithCondition
|
5
|
+
attr_accessor :attribute_description
|
6
|
+
attr_accessor :value
|
7
|
+
def initialize(attribute_description,value)
|
8
|
+
self.attribute_description=attribute_description
|
9
|
+
self.value=value
|
10
|
+
end
|
11
|
+
def matches?(domain_object)
|
12
|
+
if domain_object[attribute_description.name]==nil
|
13
|
+
return value==nil || value==:NULL
|
14
|
+
end
|
15
|
+
return domain_object[attribute_description.name].index(value)==0
|
16
|
+
end
|
17
|
+
def to_sdb_query
|
18
|
+
return "'#{self.attribute_description.name}' starts-with '#{self.attribute_description.format_for_sdb(self.value)}'"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,188 @@
|
|
1
|
+
module SdbDal
|
2
|
+
require 'sqs'
|
3
|
+
require 'memcache'
|
4
|
+
require "sdb_dal/s3.rb"
|
5
|
+
require "sdb_dal/query_string_auth_generator.rb"
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
class Storage
|
10
|
+
|
11
|
+
@conn=nil
|
12
|
+
@cache=nil
|
13
|
+
@memcache_servers=nil
|
14
|
+
@aws_key_id = nil
|
15
|
+
@aws_secret_key = nil
|
16
|
+
attr_accessor :tokens
|
17
|
+
def initialize( aws_key_id,
|
18
|
+
aws_secret_key,
|
19
|
+
memcache_servers,
|
20
|
+
tokens=[] )
|
21
|
+
@memcache_servers=memcache_servers
|
22
|
+
@aws_key_id=aws_key_id
|
23
|
+
@aws_secret_key=aws_secret_key
|
24
|
+
@tokens=tokens
|
25
|
+
if @memcache_servers and @memcache_servers.length>0
|
26
|
+
@cache= MemCache.new @memcache_servers, :namespace => 'my_namespace'
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def real_s3
|
31
|
+
unless @conn
|
32
|
+
@conn = S3::AWSAuthConnection.new(@aws_key_id, @aws_secret_key,@tokens,false)
|
33
|
+
end
|
34
|
+
return @conn
|
35
|
+
end
|
36
|
+
def real_s3_query_auth
|
37
|
+
unless @query_conn
|
38
|
+
@query_conn = S3::QueryStringAuthGenerator.new(@aws_key_id, @aws_secret_key,@tokens,false,S3::DEFAULT_HOST, 80,S3::CallingFormat::SUBDOMAIN)
|
39
|
+
end
|
40
|
+
return @query_conn
|
41
|
+
end
|
42
|
+
def renew_s3_connection
|
43
|
+
@conn=nil
|
44
|
+
real_s3
|
45
|
+
end
|
46
|
+
|
47
|
+
def encode_key(bucket,key)
|
48
|
+
val=CGI.escape(bucket+'$@#$%'+key)
|
49
|
+
if val.length>250#250 is max key length in memcached
|
50
|
+
val=val.hash.to_s
|
51
|
+
end
|
52
|
+
return val
|
53
|
+
end
|
54
|
+
def real_s3_get(bucket,key)
|
55
|
+
|
56
|
+
20.times do |i|
|
57
|
+
begin
|
58
|
+
response=real_s3.get(bucket,key)
|
59
|
+
if response and response.http_response.code=='404'
|
60
|
+
return nil
|
61
|
+
end
|
62
|
+
if response and response.http_response.code=='200'
|
63
|
+
return response.object.data
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
renew_s3_connection
|
68
|
+
|
69
|
+
rescue=> e
|
70
|
+
s= "#{e.message}\n#{e.backtrace}"
|
71
|
+
puts(s)
|
72
|
+
puts "retrying s3 get #{i.to_s}"
|
73
|
+
|
74
|
+
sleep(i*i)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
return nil
|
78
|
+
end
|
79
|
+
def create_bucket(bucket,headers={})
|
80
|
+
real_s3.create_bucket(bucket,headers)
|
81
|
+
end
|
82
|
+
def real_s3_put(bucket,key,object,attributes)
|
83
|
+
|
84
|
+
20.times do |i|
|
85
|
+
begin
|
86
|
+
real_s3.put(bucket,key,S3::S3Object.new(object),attributes)
|
87
|
+
|
88
|
+
return
|
89
|
+
|
90
|
+
rescue =>e
|
91
|
+
s= "#{e.message}\n#{e.backtrace}"
|
92
|
+
puts(s)
|
93
|
+
puts "retrying s3 put #{i.to_s}"
|
94
|
+
sleep(i*i)
|
95
|
+
#try again
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
def delete(bucket,key)
|
100
|
+
if @cache
|
101
|
+
begin
|
102
|
+
@cache[encode_key(bucket,key)]=nil
|
103
|
+
rescue
|
104
|
+
#memcache might be down
|
105
|
+
end
|
106
|
+
end
|
107
|
+
real_s3.delete(bucket,key)
|
108
|
+
|
109
|
+
end
|
110
|
+
def get(bucket,key)
|
111
|
+
value =nil
|
112
|
+
if@cache
|
113
|
+
begin
|
114
|
+
value=@cache[encode_key(bucket,key)]
|
115
|
+
rescue=>e
|
116
|
+
s= "#{e.message}"
|
117
|
+
|
118
|
+
puts("error on /#{bucket}/#{key} from cache.\n#{s}\n")
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
if !value
|
123
|
+
value=real_s3_get(bucket,key)
|
124
|
+
if @cache
|
125
|
+
begin
|
126
|
+
@cache[encode_key(bucket,key)]=value
|
127
|
+
rescue
|
128
|
+
#might be too large or memcache might be down
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
return value
|
135
|
+
|
136
|
+
end
|
137
|
+
def put(bucket,key,object,attributes={})
|
138
|
+
real_s3_put(bucket,key,object,attributes)
|
139
|
+
|
140
|
+
#cache in memcache if not image
|
141
|
+
if !attributes ||
|
142
|
+
!attributes.has_key?('Content-Type') ||
|
143
|
+
(attributes['Content-Type'].index('image')!=0 && attributes['Content-Type'].index('audio')!=0 && attributes['Content-Type'].index('video')!=0 )
|
144
|
+
if @cache
|
145
|
+
begin
|
146
|
+
@cache[encode_key(bucket,key)]=object
|
147
|
+
|
148
|
+
rescue=>e
|
149
|
+
s= "#{e.message}\n"
|
150
|
+
|
151
|
+
puts("ERROR when putting /#{bucket}/#{key}into cache.\n#{s}\n-----------------------\n")
|
152
|
+
#TODO try to whack any old value
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def list_bucket(bucket,prefix=nil)
|
159
|
+
options={}
|
160
|
+
options[:prefix]=prefix if prefix
|
161
|
+
real_s3.list_bucket(bucket,options)
|
162
|
+
|
163
|
+
end
|
164
|
+
def create_public_url(bucket,key)
|
165
|
+
return "http://s3.amazonaws.com/"+bucket+"/"+key
|
166
|
+
end
|
167
|
+
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
168
|
+
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
169
|
+
real_s3_query_auth.get(bucket,key)
|
170
|
+
|
171
|
+
|
172
|
+
end
|
173
|
+
def create_direct_url(bucket,key,time_to_live_minutes=60)
|
174
|
+
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
175
|
+
real_s3_query_auth.get(bucket,key)
|
176
|
+
|
177
|
+
|
178
|
+
end
|
179
|
+
def create_list_bucket_url(bucket,time_to_live_minutes=60)
|
180
|
+
real_s3_query_auth.expires_in=time_to_live_minutes*60
|
181
|
+
real_s3_query_auth.list_bucket(bucket)
|
182
|
+
|
183
|
+
|
184
|
+
end
|
185
|
+
|
186
|
+
end
|
187
|
+
|
188
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# TagCloud.rb
|
2
|
+
# July 6, 2007
|
3
|
+
#
|
4
|
+
|
5
|
+
class TagCloud
|
6
|
+
def calculate_max_min
|
7
|
+
@max=nil
|
8
|
+
@min=nil
|
9
|
+
for key in @histogram.keys
|
10
|
+
if @min==nil || @histogram[key]<@min
|
11
|
+
@min=@histogram[key]
|
12
|
+
end
|
13
|
+
if @max==nil || @histogram[key]>@max
|
14
|
+
@max=@histogram[key]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
def initialize(tag_data)
|
19
|
+
|
20
|
+
@histogram=tag_data.inject(Hash.new(0)){|hash,x| hash[x.tag_name]+=1;hash}
|
21
|
+
|
22
|
+
@tag_data=tag_data.sort_by{|x|[@histogram[x.tag_name],x.tag_name]}
|
23
|
+
calculate_max_min
|
24
|
+
while @histogram.size>40
|
25
|
+
@histogram.reject!{|key, value| value == @min }
|
26
|
+
calculate_max_min
|
27
|
+
end
|
28
|
+
@normal_histogram={}
|
29
|
+
if tag_data.length>0
|
30
|
+
range=@max-@min
|
31
|
+
for key in @histogram.keys
|
32
|
+
if range==0
|
33
|
+
@normal_histogram[key]=2
|
34
|
+
else
|
35
|
+
@normal_histogram[key]=((@histogram[key]-@min)*4)/range
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
@tags=@histogram.keys
|
40
|
+
@tags.sort!
|
41
|
+
else
|
42
|
+
@tags=[]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
def tags
|
46
|
+
return @tags
|
47
|
+
end
|
48
|
+
def get_count(tag_name)
|
49
|
+
return @histogram[tag_name]
|
50
|
+
end
|
51
|
+
def get_magnitude(tag_name)
|
52
|
+
return @normal_histogram[tag_name]
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module SdbDal
|
2
|
+
|
3
|
+
class TrackerDescription < DomainAttributeDescription
|
4
|
+
include SdbFormatter
|
5
|
+
attr_accessor :name
|
6
|
+
attr_accessor :other_class
|
7
|
+
attr_accessor :refecting_attribute
|
8
|
+
|
9
|
+
|
10
|
+
def initialize(name,other_class,refecting_attribute)
|
11
|
+
self.name=name
|
12
|
+
self.other_class=other_class
|
13
|
+
self.refecting_attribute=refecting_attribute
|
14
|
+
self.is_collection=true
|
15
|
+
self.is_primary_key=false
|
16
|
+
end
|
17
|
+
|
18
|
+
def format_for_sdb(value)
|
19
|
+
return value;
|
20
|
+
end
|
21
|
+
def parse_from_sdb( value)
|
22
|
+
return value
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
data/lib/sdb_dal.rb
ADDED
@@ -0,0 +1,8 @@
|
|
1
|
+
require "sdb_dal/repository.rb"
|
2
|
+
require "sdb_dal/memory_repository.rb"
|
3
|
+
require "sdb_dal/domain_object.rb"
|
4
|
+
require "sdb_dal/storage.rb"
|
5
|
+
require "sdb_dal/s3.rb"
|
6
|
+
require "sdb_dal/attribute_range.rb"
|
7
|
+
require "sdb_dal/and_condition.rb"
|
8
|
+
require "sdb_dal/or_condition.rb"
|
metadata
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sdb_dal
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Knight
|
8
|
+
autorequire: sdb_dal
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2008-10-31 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies: []
|
15
|
+
|
16
|
+
description: SDB DAL.
|
17
|
+
email: knight@davideknight.com
|
18
|
+
executables: []
|
19
|
+
|
20
|
+
extensions: []
|
21
|
+
|
22
|
+
extra_rdoc_files: []
|
23
|
+
|
24
|
+
files:
|
25
|
+
- ./lib/sdb_dal.rb
|
26
|
+
- ./lib/sdb_dal/and_condition.rb
|
27
|
+
- ./lib/sdb_dal/attribute_range.rb
|
28
|
+
- ./lib/sdb_dal/domain_attribute_description.rb
|
29
|
+
- ./lib/sdb_dal/domain_object.rb
|
30
|
+
- ./lib/sdb_dal/domain_object_cache_item.rb
|
31
|
+
- ./lib/sdb_dal/equals_condition.rb
|
32
|
+
- ./lib/sdb_dal/geo.rb
|
33
|
+
- ./lib/sdb_dal/index_description.rb
|
34
|
+
- ./lib/sdb_dal/is_null_transform.rb
|
35
|
+
- ./lib/sdb_dal/lazy_loading_text.rb
|
36
|
+
- ./lib/sdb_dal/memory_repository.rb
|
37
|
+
- ./lib/sdb_dal/memory_storage.rb
|
38
|
+
- ./lib/sdb_dal/or_condition.rb
|
39
|
+
- ./lib/sdb_dal/query_string_auth_generator.rb
|
40
|
+
- ./lib/sdb_dal/reference.rb
|
41
|
+
- ./lib/sdb_dal/repository.rb
|
42
|
+
- ./lib/sdb_dal/s3.rb
|
43
|
+
- ./lib/sdb_dal/sdb_formatter.rb
|
44
|
+
- ./lib/sdb_dal/starts_with_condition.rb
|
45
|
+
- ./lib/sdb_dal/storage.rb
|
46
|
+
- ./lib/sdb_dal/tag_cloud.rb
|
47
|
+
- ./lib/sdb_dal/tracker_description.rb
|
48
|
+
has_rdoc: false
|
49
|
+
homepage: http://www.yahoo.com/
|
50
|
+
post_install_message:
|
51
|
+
rdoc_options: []
|
52
|
+
|
53
|
+
require_paths:
|
54
|
+
- lib
|
55
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: "0"
|
60
|
+
version:
|
61
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
62
|
+
requirements:
|
63
|
+
- - ">="
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: "0"
|
66
|
+
version:
|
67
|
+
requirements: []
|
68
|
+
|
69
|
+
rubyforge_project:
|
70
|
+
rubygems_version: 1.3.1
|
71
|
+
signing_key:
|
72
|
+
specification_version: 2
|
73
|
+
summary: DAL Layer for Amazon's SDB.
|
74
|
+
test_files: []
|
75
|
+
|