ebx_deliver 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/.gitignore +22 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +55 -0
- data/LICENSE +20 -0
- data/README.md +35 -0
- data/Rakefile +1 -0
- data/ebx_deliver.gemspec +27 -0
- data/lib/dynamoid/adapter/aws_multi_region_adapter.rb +281 -0
- data/lib/ebx_deliver.rb +11 -0
- data/lib/ebx_deliver/aws_sns_writer.rb +100 -0
- data/lib/ebx_deliver/aws_sqs_reader.rb +55 -0
- data/lib/ebx_deliver/version.rb +3 -0
- metadata +129 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZDVhYzJlMzdkOWYzMDFiZDEzM2MwN2RmMGU3MzJkNGI2YTc4MzIxYw==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ODc4ODQwNTdhNjMxNWY3NDJmNWNkNDAxNDU1NzNlOTFkZWM3Mjg1Zg==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MTk4ZjdlNjJkNzUyNWU1NWQzOTA5MmNhMjVlMDc2YjQzYTBkM2NjN2FlNWQ3
|
10
|
+
NmU5NTUwNzg0ZDY3YmM1YmM5MDU3YTVjODhiNGZiNGM1ZDkyMjNhNDY5YzFk
|
11
|
+
ZWQyY2I0NTUzOWQ2NzQxNzU3ZjJjNmI2YmJkYWYxNzVkNjUwOGI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YzdhOGM2ODRmZDgxYjAwMjg3OGNmZmQyY2Y1MmJlYjFkYjYyYTgyYTFiYWI0
|
14
|
+
YWYwYjIwNzkzODZlZGExZTIzNmY2YzlmZmRkYWRkZmUwNTQ1Y2E4NmJiMTFj
|
15
|
+
OGJkYjg3ZmFjODkwN2Y5MTIyMGM2MTEyNjdiN2VlNTc3MTY2MmE=
|
data/.gitignore
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.rvmrc
|
4
|
+
.idea
|
5
|
+
.bundle
|
6
|
+
.config
|
7
|
+
coverage
|
8
|
+
InstalledFiles
|
9
|
+
lib/bundler/man
|
10
|
+
pkg
|
11
|
+
rdoc
|
12
|
+
spec/reports
|
13
|
+
test/tmp
|
14
|
+
test/version_tmp
|
15
|
+
tmp
|
16
|
+
|
17
|
+
# YARD artifacts
|
18
|
+
.yardoc
|
19
|
+
_yardoc
|
20
|
+
doc/
|
21
|
+
.DS_Store
|
22
|
+
eb/
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby 1.9.3
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
ebx_deliver (0.0.1)
|
5
|
+
aws-sdk (~> 1.17.0)
|
6
|
+
dynamoid (~> 0.7.1)
|
7
|
+
pry (> 0)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
activemodel (4.0.0)
|
13
|
+
activesupport (= 4.0.0)
|
14
|
+
builder (~> 3.1.0)
|
15
|
+
activesupport (4.0.0)
|
16
|
+
i18n (~> 0.6, >= 0.6.4)
|
17
|
+
minitest (~> 4.2)
|
18
|
+
multi_json (~> 1.3)
|
19
|
+
thread_safe (~> 0.1)
|
20
|
+
tzinfo (~> 0.3.37)
|
21
|
+
atomic (1.1.13)
|
22
|
+
aws-sdk (1.17.0)
|
23
|
+
json (~> 1.4)
|
24
|
+
nokogiri (< 1.6.0)
|
25
|
+
uuidtools (~> 2.1)
|
26
|
+
builder (3.1.4)
|
27
|
+
coderay (1.0.9)
|
28
|
+
dynamoid (0.7.1)
|
29
|
+
activemodel
|
30
|
+
aws-sdk
|
31
|
+
tzinfo
|
32
|
+
i18n (0.6.5)
|
33
|
+
json (1.8.0)
|
34
|
+
method_source (0.8.2)
|
35
|
+
minitest (4.7.5)
|
36
|
+
multi_json (1.8.0)
|
37
|
+
nokogiri (1.5.10)
|
38
|
+
pry (0.9.12.2)
|
39
|
+
coderay (~> 1.0.5)
|
40
|
+
method_source (~> 0.8)
|
41
|
+
slop (~> 3.4)
|
42
|
+
rake (10.1.0)
|
43
|
+
slop (3.4.6)
|
44
|
+
thread_safe (0.1.3)
|
45
|
+
atomic
|
46
|
+
tzinfo (0.3.37)
|
47
|
+
uuidtools (2.1.4)
|
48
|
+
|
49
|
+
PLATFORMS
|
50
|
+
ruby
|
51
|
+
|
52
|
+
DEPENDENCIES
|
53
|
+
bundler (~> 1.3)
|
54
|
+
ebx_deliver!
|
55
|
+
rake
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Funnerator Enterprises, Inc.
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# EbxDeliver
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'ebx_deliver'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install ebx_deliver
|
18
|
+
|
19
|
+
Configure Dynamoid
|
20
|
+
|
21
|
+
Dynamoid.configure do |config|
|
22
|
+
config.adapter = 'aws_multi_region_adapter'
|
23
|
+
end
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
TODO: Write usage instructions here
|
28
|
+
|
29
|
+
## Contributing
|
30
|
+
|
31
|
+
1. Fork it
|
32
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
33
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
34
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
35
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/ebx_deliver.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ebx_deliver/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ebx_deliver"
|
8
|
+
spec.version = EbxDeliver::VERSION
|
9
|
+
spec.authors = ["Alex Bullard"]
|
10
|
+
spec.email = ["abullrd@gmail.com"]
|
11
|
+
spec.description = "eb eXtended cross region db command distributor"
|
12
|
+
spec.summary = "Distributes writes to Amazon's DynamoDB through SNS to allow for multiple cross region databases"
|
13
|
+
spec.homepage = "https://github.com/Funnerator/ebx_deliver.git"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = []
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
|
24
|
+
spec.add_runtime_dependency("dynamoid", "~> 0.7.1")
|
25
|
+
spec.add_runtime_dependency("aws-sdk", "~> 1.17.0")
|
26
|
+
spec.add_runtime_dependency("pry", "> 0")
|
27
|
+
end
|
@@ -0,0 +1,281 @@
|
|
1
|
+
module Dynamoid
|
2
|
+
module Adapter
|
3
|
+
module AwsMultiRegionAdapter
|
4
|
+
extend self
|
5
|
+
@@connection = nil
|
6
|
+
|
7
|
+
# Establish the connection to DynamoDB.
|
8
|
+
#
|
9
|
+
# @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
|
10
|
+
# Call DynamoDB new, with no parameters.
|
11
|
+
# Make sure the aws.yml file or aws.rb file, refer the link for more details.
|
12
|
+
#https://github.com/amazonwebservices/aws-sdk-for-ruby
|
13
|
+
# 1. Create config/aws.yml as follows:
|
14
|
+
# Fill in your AWS Access Key ID and Secret Access Key
|
15
|
+
# http://aws.amazon.com/security-credentials
|
16
|
+
#access_key_id: REPLACE_WITH_ACCESS_KEY_ID
|
17
|
+
#secret_access_key: REPLACE_WITH_SECRET_ACCESS_KEY
|
18
|
+
#(or)
|
19
|
+
#2, Create config/initializers/aws.rb as follows:
|
20
|
+
# load the libraries
|
21
|
+
#require 'aws'
|
22
|
+
# log requests using the default rails logger
|
23
|
+
#AWS.config(:logger => Rails.logger)
|
24
|
+
# load credentials from a file
|
25
|
+
#config_path = File.expand_path(File.dirname(__FILE__)+"/../aws.yml")
|
26
|
+
#AWS.config(YAML.load(File.read(config_path)))
|
27
|
+
#Additionally include any of the dynamodb paramters as needed
|
28
|
+
#(eg: if you would like to change the dynamodb endpoint, then add the parameter in
|
29
|
+
# the following paramter in the file aws.yml or aws.rb
|
30
|
+
# dynamo_db_endpoint : dynamodb.ap-southeast-1.amazonaws.com)
|
31
|
+
# @since 0.2.0
|
32
|
+
def connect!
|
33
|
+
@@connection = ::EbxDeliver::AwsSnsWriter.new
|
34
|
+
end
|
35
|
+
|
36
|
+
# Return the established connection.
|
37
|
+
#
|
38
|
+
# @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
|
39
|
+
#
|
40
|
+
# @since 0.2.0
|
41
|
+
def connection
|
42
|
+
@@connection
|
43
|
+
end
|
44
|
+
|
45
|
+
# Get many items at once from DynamoDB. More efficient than getting each item individually.
|
46
|
+
#
|
47
|
+
# @example Retrieve IDs 1 and 2 from the table testtable
|
48
|
+
# Dynamoid::Adapter::AwsSdk.batch_get_item({'table1' => ['1', '2']}, :consistent_read => true)
|
49
|
+
#
|
50
|
+
# @param [Hash] table_ids the hash of tables and IDs to retrieve
|
51
|
+
# @param [Hash] options to be passed to underlying BatchGet call
|
52
|
+
#
|
53
|
+
# @return [Hash] a hash where keys are the table names and the values are the retrieved items
|
54
|
+
#
|
55
|
+
# @since 0.2.0
|
56
|
+
def batch_get_item(table_ids, options = {})
|
57
|
+
binding.pry
|
58
|
+
connection.batch_get do |batch|
|
59
|
+
table_ids.each do |table, keys|
|
60
|
+
batch.table(table, :all, keys)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
|
66
|
+
#return hash if table_ids.all?{|k, v| v.empty?}
|
67
|
+
#table_ids.each do |t, ids|
|
68
|
+
# Array(ids).in_groups_of(100, false) do |group|
|
69
|
+
# batch = AWS::DynamoDB::BatchGet.new(:config => @@connection.config)
|
70
|
+
# batch.table(t, :all, Array(group), options) unless group.nil? || group.empty?
|
71
|
+
# batch.each do |table_name, attributes|
|
72
|
+
# hash[table_name] << attributes.symbolize_keys!
|
73
|
+
# end
|
74
|
+
# end
|
75
|
+
#end
|
76
|
+
#hash
|
77
|
+
end
|
78
|
+
|
79
|
+
# Delete many items at once from DynamoDB. More efficient than delete each item individually.
|
80
|
+
#
|
81
|
+
# @example Delete IDs 1 and 2 from the table testtable
|
82
|
+
# Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => ['1', '2'])
|
83
|
+
#or
|
84
|
+
# Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
|
85
|
+
#
|
86
|
+
# @param [Hash] options the hash of tables and IDs to delete
|
87
|
+
#
|
88
|
+
# @return nil
|
89
|
+
#
|
90
|
+
def batch_delete_item(options)
|
91
|
+
return nil if options.all?{|k, v| v.empty?}
|
92
|
+
options.each do |t, ids|
|
93
|
+
Array(ids).in_groups_of(25, false) do |group|
|
94
|
+
batch = AWS::DynamoDB::BatchWrite.new(:config => @@connection.config)
|
95
|
+
batch.delete(t,group)
|
96
|
+
batch.process!
|
97
|
+
end
|
98
|
+
end
|
99
|
+
nil
|
100
|
+
end
|
101
|
+
|
102
|
+
# Create a table on DynamoDB. This usually takes a long time to complete.
|
103
|
+
#
|
104
|
+
# @param [String] table_name the name of the table to create
|
105
|
+
# @param [Symbol] key the table's primary key (defaults to :id)
|
106
|
+
# @param [Hash] options provide a range_key here if you want one for the table
|
107
|
+
#
|
108
|
+
# @since 0.2.0
|
109
|
+
def create_table(table_name, key = :id, options = {})
|
110
|
+
Dynamoid.logger.info "Creating #{table_name} table. This could take a while."
|
111
|
+
options[:hash_key] ||= {key.to_sym => :string}
|
112
|
+
read_capacity = options[:read_capacity] || Dynamoid::Config.read_capacity
|
113
|
+
write_capacity = options[:write_capacity] || Dynamoid::Config.write_capacity
|
114
|
+
table = @@connection.tables.create(table_name, read_capacity, write_capacity, options)
|
115
|
+
sleep 0.5 while table.status == :creating
|
116
|
+
return table
|
117
|
+
end
|
118
|
+
|
119
|
+
# Removes an item from DynamoDB.
|
120
|
+
#
|
121
|
+
# @param [String] table_name the name of the table
|
122
|
+
# @param [String] key the hash key of the item to delete
|
123
|
+
# @param [Number] range_key the range key of the item to delete, required if the table has a composite key
|
124
|
+
#
|
125
|
+
# @since 0.2.0
|
126
|
+
def delete_item(table_name, key, options = {})
|
127
|
+
range_key = options.delete(:range_key)
|
128
|
+
table = get_table(table_name)
|
129
|
+
result = table.items.at(key, range_key)
|
130
|
+
result.delete unless result.attributes.to_h.empty?
|
131
|
+
true
|
132
|
+
end
|
133
|
+
|
134
|
+
# Deletes an entire table from DynamoDB. Only 10 tables can be in the deleting state at once,
|
135
|
+
# so if you have more this method may raise an exception.
|
136
|
+
#
|
137
|
+
# @param [String] table_name the name of the table to destroy
|
138
|
+
#
|
139
|
+
# @since 0.2.0
|
140
|
+
def delete_table(table_name)
|
141
|
+
Dynamoid.logger.info "Deleting #{table_name} table. This could take a while."
|
142
|
+
table = @@connection.tables[table_name]
|
143
|
+
table.delete
|
144
|
+
sleep 0.5 while table.exists? == true
|
145
|
+
end
|
146
|
+
|
147
|
+
# @todo Add a DescribeTable method.
|
148
|
+
|
149
|
+
# Fetches an item from DynamoDB.
|
150
|
+
#
|
151
|
+
# @param [String] table_name the name of the table
|
152
|
+
# @param [String] key the hash key of the item to find
|
153
|
+
# @param [Number] range_key the range key of the item to find, required if the table has a composite key
|
154
|
+
#
|
155
|
+
# @return [Hash] a hash representing the raw item in DynamoDB
|
156
|
+
#
|
157
|
+
# @since 0.2.0
|
158
|
+
def get_item(table_name, key, options = {})
|
159
|
+
range_key = options.delete(:range_key)
|
160
|
+
table = get_table(table_name)
|
161
|
+
|
162
|
+
result = table.items.at(key, range_key).attributes.to_h(options)
|
163
|
+
|
164
|
+
if result.empty?
|
165
|
+
nil
|
166
|
+
else
|
167
|
+
result.symbolize_keys!
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def update_item(table_name, key, options = {}, &block)
|
172
|
+
range_key = options.delete(:range_key)
|
173
|
+
conditions = options.delete(:conditions) || {}
|
174
|
+
table = get_table(table_name)
|
175
|
+
item = table.items.at(key, range_key)
|
176
|
+
item.attributes.update(conditions.merge(:return => :all_new), &block)
|
177
|
+
rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException
|
178
|
+
raise Dynamoid::Errors::ConditionalCheckFailedException
|
179
|
+
end
|
180
|
+
|
181
|
+
# List all tables on DynamoDB.
|
182
|
+
#
|
183
|
+
# @since 0.2.0
|
184
|
+
def list_tables
|
185
|
+
connection.db.tables.collect(&:name)
|
186
|
+
end
|
187
|
+
|
188
|
+
# Persists an item on DynamoDB.
|
189
|
+
#
|
190
|
+
# @param [String] table_name the name of the table
|
191
|
+
# @param [Object] object a hash or Dynamoid object to persist
|
192
|
+
#
|
193
|
+
# @since 0.2.0
|
194
|
+
def put_item(table_name, object, options = nil)
|
195
|
+
connection.put_item(table_name, object, options)
|
196
|
+
end
|
197
|
+
|
198
|
+
# Query the DynamoDB table. This employs DynamoDB's indexes so is generally faster than scanning, but is
|
199
|
+
# only really useful for range queries, since it can only find by one hash key at once. Only provide
|
200
|
+
# one range key to the hash.
|
201
|
+
#
|
202
|
+
# @param [String] table_name the name of the table
|
203
|
+
# @param [Hash] opts the options to query the table with
|
204
|
+
# @option opts [String] :hash_value the value of the hash key to find
|
205
|
+
# @option opts [Range] :range_value find the range key within this range
|
206
|
+
# @option opts [Number] :range_greater_than find range keys greater than this
|
207
|
+
# @option opts [Number] :range_less_than find range keys less than this
|
208
|
+
# @option opts [Number] :range_gte find range keys greater than or equal to this
|
209
|
+
# @option opts [Number] :range_lte find range keys less than or equal to this
|
210
|
+
#
|
211
|
+
# @return [Enumerator] an iterator of all matching items
|
212
|
+
#
|
213
|
+
# @since 0.2.0
|
214
|
+
def query(table_name, opts = {})
|
215
|
+
table = get_table(table_name)
|
216
|
+
|
217
|
+
Enumerator.new do |yielder|
|
218
|
+
consistent_opts = { :consistent_read => opts[:consistent_read] || false }
|
219
|
+
if table.composite_key?
|
220
|
+
table.items.query(opts).each do |data|
|
221
|
+
if opts.has_key? :select
|
222
|
+
yielder.yield data.attributes.symbolize_keys!
|
223
|
+
else
|
224
|
+
yielder.yield data.attributes.to_h(consistent_opts).symbolize_keys!
|
225
|
+
end
|
226
|
+
end
|
227
|
+
else
|
228
|
+
yielder.yield get_item(table_name, opts[:hash_value])
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
# Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on
|
234
|
+
# the DynamoDB servers.
|
235
|
+
#
|
236
|
+
# @param [String] table_name the name of the table
|
237
|
+
# @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
|
238
|
+
#
|
239
|
+
# @return [Enumerator] an iterator of all matching items
|
240
|
+
#
|
241
|
+
# @since 0.2.0
|
242
|
+
def scan(table_name, scan_hash, select_opts)
|
243
|
+
table = get_table(table_name)
|
244
|
+
Enumerator.new do |yielder|
|
245
|
+
table.items.where(scan_hash).select(select_opts).each do |data|
|
246
|
+
yielder.yield data.attributes.symbolize_keys!
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
# @todo Add an UpdateItem method.
|
252
|
+
|
253
|
+
# @todo Add an UpdateTable method.
|
254
|
+
|
255
|
+
def get_table(table_name)
|
256
|
+
unless table = table_cache[table_name]
|
257
|
+
table = @@connection.db.tables[table_name]
|
258
|
+
table.load_schema
|
259
|
+
table_cache[table_name] = table
|
260
|
+
end
|
261
|
+
table
|
262
|
+
end
|
263
|
+
|
264
|
+
def table_cache
|
265
|
+
@table_cache ||= {}
|
266
|
+
end
|
267
|
+
|
268
|
+
# Number of items from a table
|
269
|
+
#
|
270
|
+
# @param [String] table_name the name of the table
|
271
|
+
#
|
272
|
+
# @return [Integer] the number of items from a table
|
273
|
+
#
|
274
|
+
# @since 0.6.1
|
275
|
+
def count(table_name)
|
276
|
+
table = get_table(table_name)
|
277
|
+
table.items.count
|
278
|
+
end
|
279
|
+
end
|
280
|
+
end
|
281
|
+
end
|
data/lib/ebx_deliver.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
module EbxDeliver
|
2
|
+
class AwsSnsWriter
|
3
|
+
attr_accessor :db
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
# TODO fix
|
7
|
+
@topic = AWS.sns.topics.create('development-sns')
|
8
|
+
|
9
|
+
@db = AWS::DynamoDB.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def batch_get_item(&block)
|
13
|
+
@db.batch_get_item(&block)
|
14
|
+
end
|
15
|
+
|
16
|
+
def batch_write_item(options)
|
17
|
+
@db.client.batch_write_item(options)
|
18
|
+
@topic.publish({
|
19
|
+
method: 'batch_write_item',
|
20
|
+
args: [options]
|
21
|
+
}.to_json)
|
22
|
+
end
|
23
|
+
|
24
|
+
def batch_delete_item(options)
|
25
|
+
@topic.publish({
|
26
|
+
method: 'batch_delete_item',
|
27
|
+
args: [options]
|
28
|
+
}.to_json)
|
29
|
+
end
|
30
|
+
|
31
|
+
def create_table(options = {})
|
32
|
+
@topic.publish({
|
33
|
+
method: 'create_table',
|
34
|
+
args: [options]
|
35
|
+
}.to_json)
|
36
|
+
end
|
37
|
+
|
38
|
+
def delete_item(options = {})
|
39
|
+
@topic.publish({
|
40
|
+
method: 'delete_item',
|
41
|
+
args: [options]
|
42
|
+
}.to_json)
|
43
|
+
end
|
44
|
+
|
45
|
+
def delete_table(options = {})
|
46
|
+
@topic.publish({
|
47
|
+
method: 'delete_table',
|
48
|
+
args: [table_name]
|
49
|
+
}.to_json)
|
50
|
+
end
|
51
|
+
|
52
|
+
def describe_table(options = {})
|
53
|
+
@db.client.describe_table(options)
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_item(table_name, key, options = {})
|
57
|
+
@db.client.get_item(options)
|
58
|
+
end
|
59
|
+
|
60
|
+
def list_tables
|
61
|
+
@db.client.list_tables(options)
|
62
|
+
end
|
63
|
+
|
64
|
+
def put_item(table_name, object, options = nil)
|
65
|
+
@topic.publish({
|
66
|
+
method: 'put_item',
|
67
|
+
args: [table_name, object, options]
|
68
|
+
}.to_json)
|
69
|
+
end
|
70
|
+
|
71
|
+
def query(table_name, opts = {})
|
72
|
+
@db.client.query(options)
|
73
|
+
end
|
74
|
+
|
75
|
+
def scan(options)
|
76
|
+
@db.client.scan(options)
|
77
|
+
end
|
78
|
+
|
79
|
+
def update_item(options = {})
|
80
|
+
@topic.publish({
|
81
|
+
method: 'update_item',
|
82
|
+
args: [options]
|
83
|
+
}.to_json)
|
84
|
+
end
|
85
|
+
|
86
|
+
def update_table(options = {})
|
87
|
+
@topic.publish({
|
88
|
+
method: 'update_item',
|
89
|
+
args: [options]
|
90
|
+
}.to_json)
|
91
|
+
end
|
92
|
+
|
93
|
+
def self.send_test
|
94
|
+
w = AwsSnsWriter.new
|
95
|
+
w.connect!
|
96
|
+
|
97
|
+
w.create_table('testtt')
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module EbxDeliver
|
2
|
+
class AwsSqsReader
|
3
|
+
def connect!
|
4
|
+
# TODO pull name from settings
|
5
|
+
@queue = sqs.queues.named('development-sns')
|
6
|
+
|
7
|
+
@db = AWS::DynamoDB.new
|
8
|
+
|
9
|
+
@queue.poll do |notification|
|
10
|
+
msg = JSON.parse(JSON.parse(notification.body)['Message'])
|
11
|
+
Rails.logger.info "MSG Received #{msg}"
|
12
|
+
send(msg['method'], *msg['args'])
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_table(options)
|
17
|
+
@db.client.create_table(options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def batch_delete_item(options)
|
21
|
+
@db.client.batch_delete_item(options)
|
22
|
+
end
|
23
|
+
|
24
|
+
def create_table(options = {})
|
25
|
+
@db.client.create_table(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def delete_item(options = {})
|
29
|
+
@db.client.delete_item(options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete_table(options = {})
|
33
|
+
@db.client.delete_table(options)
|
34
|
+
end
|
35
|
+
|
36
|
+
def put_item(table_name, object, options = {})
|
37
|
+
table = @db.tables[table_name]
|
38
|
+
table.load_schema
|
39
|
+
table.items.create(
|
40
|
+
object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)},
|
41
|
+
options || {}
|
42
|
+
)
|
43
|
+
rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
|
44
|
+
raise Dynamoid::Errors::ConditionalCheckFailedException
|
45
|
+
end
|
46
|
+
|
47
|
+
def update_item(options = {})
|
48
|
+
@db.client.update_item(options)
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_table(options = {})
|
52
|
+
@db.client.update_table(options)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
metadata
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ebx_deliver
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alex Bullard
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-10-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: dynamoid
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.7.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.7.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.17.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.17.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ! '>'
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ! '>'
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: eb eXtended cross region db command distributor
|
84
|
+
email:
|
85
|
+
- abullrd@gmail.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- .gitignore
|
91
|
+
- .ruby-version
|
92
|
+
- Gemfile
|
93
|
+
- Gemfile.lock
|
94
|
+
- LICENSE
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- ebx_deliver.gemspec
|
98
|
+
- lib/dynamoid/adapter/aws_multi_region_adapter.rb
|
99
|
+
- lib/ebx_deliver.rb
|
100
|
+
- lib/ebx_deliver/aws_sns_writer.rb
|
101
|
+
- lib/ebx_deliver/aws_sqs_reader.rb
|
102
|
+
- lib/ebx_deliver/version.rb
|
103
|
+
homepage: https://github.com/Funnerator/ebx_deliver.git
|
104
|
+
licenses:
|
105
|
+
- MIT
|
106
|
+
metadata: {}
|
107
|
+
post_install_message:
|
108
|
+
rdoc_options: []
|
109
|
+
require_paths:
|
110
|
+
- lib
|
111
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - ! '>='
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
116
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - ! '>='
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
121
|
+
requirements: []
|
122
|
+
rubyforge_project:
|
123
|
+
rubygems_version: 2.0.6
|
124
|
+
signing_key:
|
125
|
+
specification_version: 4
|
126
|
+
summary: Distributes writes to Amazon's DynamoDB through SNS to allow for multiple
|
127
|
+
cross region databases
|
128
|
+
test_files: []
|
129
|
+
has_rdoc:
|