ebx_deliver 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/.gitignore +22 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +55 -0
- data/LICENSE +20 -0
- data/README.md +35 -0
- data/Rakefile +1 -0
- data/ebx_deliver.gemspec +27 -0
- data/lib/dynamoid/adapter/aws_multi_region_adapter.rb +281 -0
- data/lib/ebx_deliver.rb +11 -0
- data/lib/ebx_deliver/aws_sns_writer.rb +100 -0
- data/lib/ebx_deliver/aws_sqs_reader.rb +55 -0
- data/lib/ebx_deliver/version.rb +3 -0
- metadata +129 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZDVhYzJlMzdkOWYzMDFiZDEzM2MwN2RmMGU3MzJkNGI2YTc4MzIxYw==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ODc4ODQwNTdhNjMxNWY3NDJmNWNkNDAxNDU1NzNlOTFkZWM3Mjg1Zg==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MTk4ZjdlNjJkNzUyNWU1NWQzOTA5MmNhMjVlMDc2YjQzYTBkM2NjN2FlNWQ3
|
10
|
+
NmU5NTUwNzg0ZDY3YmM1YmM5MDU3YTVjODhiNGZiNGM1ZDkyMjNhNDY5YzFk
|
11
|
+
ZWQyY2I0NTUzOWQ2NzQxNzU3ZjJjNmI2YmJkYWYxNzVkNjUwOGI=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
YzdhOGM2ODRmZDgxYjAwMjg3OGNmZmQyY2Y1MmJlYjFkYjYyYTgyYTFiYWI0
|
14
|
+
YWYwYjIwNzkzODZlZGExZTIzNmY2YzlmZmRkYWRkZmUwNTQ1Y2E4NmJiMTFj
|
15
|
+
OGJkYjg3ZmFjODkwN2Y5MTIyMGM2MTEyNjdiN2VlNTc3MTY2MmE=
|
data/.gitignore
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
.rvmrc
|
4
|
+
.idea
|
5
|
+
.bundle
|
6
|
+
.config
|
7
|
+
coverage
|
8
|
+
InstalledFiles
|
9
|
+
lib/bundler/man
|
10
|
+
pkg
|
11
|
+
rdoc
|
12
|
+
spec/reports
|
13
|
+
test/tmp
|
14
|
+
test/version_tmp
|
15
|
+
tmp
|
16
|
+
|
17
|
+
# YARD artifacts
|
18
|
+
.yardoc
|
19
|
+
_yardoc
|
20
|
+
doc/
|
21
|
+
.DS_Store
|
22
|
+
eb/
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby 1.9.3
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
ebx_deliver (0.0.1)
|
5
|
+
aws-sdk (~> 1.17.0)
|
6
|
+
dynamoid (~> 0.7.1)
|
7
|
+
pry (> 0)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
activemodel (4.0.0)
|
13
|
+
activesupport (= 4.0.0)
|
14
|
+
builder (~> 3.1.0)
|
15
|
+
activesupport (4.0.0)
|
16
|
+
i18n (~> 0.6, >= 0.6.4)
|
17
|
+
minitest (~> 4.2)
|
18
|
+
multi_json (~> 1.3)
|
19
|
+
thread_safe (~> 0.1)
|
20
|
+
tzinfo (~> 0.3.37)
|
21
|
+
atomic (1.1.13)
|
22
|
+
aws-sdk (1.17.0)
|
23
|
+
json (~> 1.4)
|
24
|
+
nokogiri (< 1.6.0)
|
25
|
+
uuidtools (~> 2.1)
|
26
|
+
builder (3.1.4)
|
27
|
+
coderay (1.0.9)
|
28
|
+
dynamoid (0.7.1)
|
29
|
+
activemodel
|
30
|
+
aws-sdk
|
31
|
+
tzinfo
|
32
|
+
i18n (0.6.5)
|
33
|
+
json (1.8.0)
|
34
|
+
method_source (0.8.2)
|
35
|
+
minitest (4.7.5)
|
36
|
+
multi_json (1.8.0)
|
37
|
+
nokogiri (1.5.10)
|
38
|
+
pry (0.9.12.2)
|
39
|
+
coderay (~> 1.0.5)
|
40
|
+
method_source (~> 0.8)
|
41
|
+
slop (~> 3.4)
|
42
|
+
rake (10.1.0)
|
43
|
+
slop (3.4.6)
|
44
|
+
thread_safe (0.1.3)
|
45
|
+
atomic
|
46
|
+
tzinfo (0.3.37)
|
47
|
+
uuidtools (2.1.4)
|
48
|
+
|
49
|
+
PLATFORMS
|
50
|
+
ruby
|
51
|
+
|
52
|
+
DEPENDENCIES
|
53
|
+
bundler (~> 1.3)
|
54
|
+
ebx_deliver!
|
55
|
+
rake
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2013 Funnerator Enterprises, Inc.
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
6
|
+
this software and associated documentation files (the "Software"), to deal in
|
7
|
+
the Software without restriction, including without limitation the rights to
|
8
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
9
|
+
the Software, and to permit persons to whom the Software is furnished to do so,
|
10
|
+
subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
17
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
18
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
19
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
20
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# EbxDeliver
|
2
|
+
|
3
|
+
TODO: Write a gem description
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'ebx_deliver'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install ebx_deliver
|
18
|
+
|
19
|
+
Configure Dynamoid
|
20
|
+
|
21
|
+
Dynamoid.configure do |config|
|
22
|
+
config.adapter = 'aws_multi_region_adapter'
|
23
|
+
end
|
24
|
+
|
25
|
+
## Usage
|
26
|
+
|
27
|
+
TODO: Write usage instructions here
|
28
|
+
|
29
|
+
## Contributing
|
30
|
+
|
31
|
+
1. Fork it
|
32
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
33
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
34
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
35
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/ebx_deliver.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ebx_deliver/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ebx_deliver"
|
8
|
+
spec.version = EbxDeliver::VERSION
|
9
|
+
spec.authors = ["Alex Bullard"]
|
10
|
+
spec.email = ["abullrd@gmail.com"]
|
11
|
+
spec.description = "eb eXtended cross region db command distributor"
|
12
|
+
spec.summary = "Distributes writes to Amazon's DynamoDB through SNS to allow for multiple cross region databases"
|
13
|
+
spec.homepage = "https://github.com/Funnerator/ebx_deliver.git"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = []
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
22
|
+
spec.add_development_dependency "rake"
|
23
|
+
|
24
|
+
spec.add_runtime_dependency("dynamoid", "~> 0.7.1")
|
25
|
+
spec.add_runtime_dependency("aws-sdk", "~> 1.17.0")
|
26
|
+
spec.add_runtime_dependency("pry", "> 0")
|
27
|
+
end
|
@@ -0,0 +1,281 @@
|
|
1
|
+
module Dynamoid
|
2
|
+
module Adapter
|
3
|
+
module AwsMultiRegionAdapter
|
4
|
+
extend self
|
5
|
+
@@connection = nil
|
6
|
+
|
7
|
+
# Establish the connection to DynamoDB.
|
8
|
+
#
|
9
|
+
# @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
|
10
|
+
# Call DynamoDB new, with no parameters.
|
11
|
+
# Make sure the aws.yml file or aws.rb file, refer the link for more details.
|
12
|
+
#https://github.com/amazonwebservices/aws-sdk-for-ruby
|
13
|
+
# 1. Create config/aws.yml as follows:
|
14
|
+
# Fill in your AWS Access Key ID and Secret Access Key
|
15
|
+
# http://aws.amazon.com/security-credentials
|
16
|
+
#access_key_id: REPLACE_WITH_ACCESS_KEY_ID
|
17
|
+
#secret_access_key: REPLACE_WITH_SECRET_ACCESS_KEY
|
18
|
+
#(or)
|
19
|
+
#2, Create config/initializers/aws.rb as follows:
|
20
|
+
# load the libraries
|
21
|
+
#require 'aws'
|
22
|
+
# log requests using the default rails logger
|
23
|
+
#AWS.config(:logger => Rails.logger)
|
24
|
+
# load credentials from a file
|
25
|
+
#config_path = File.expand_path(File.dirname(__FILE__)+"/../aws.yml")
|
26
|
+
#AWS.config(YAML.load(File.read(config_path)))
|
27
|
+
#Additionally include any of the dynamodb paramters as needed
|
28
|
+
#(eg: if you would like to change the dynamodb endpoint, then add the parameter in
|
29
|
+
# the following paramter in the file aws.yml or aws.rb
|
30
|
+
# dynamo_db_endpoint : dynamodb.ap-southeast-1.amazonaws.com)
|
31
|
+
# @since 0.2.0
|
32
|
+
def connect!
|
33
|
+
@@connection = ::EbxDeliver::AwsSnsWriter.new
|
34
|
+
end
|
35
|
+
|
36
|
+
# Return the established connection.
|
37
|
+
#
|
38
|
+
# @return [AWS::DynamoDB::Connection] the raw DynamoDB connection
|
39
|
+
#
|
40
|
+
# @since 0.2.0
|
41
|
+
def connection
|
42
|
+
@@connection
|
43
|
+
end
|
44
|
+
|
45
|
+
# Get many items at once from DynamoDB. More efficient than getting each item individually.
|
46
|
+
#
|
47
|
+
# @example Retrieve IDs 1 and 2 from the table testtable
|
48
|
+
# Dynamoid::Adapter::AwsSdk.batch_get_item({'table1' => ['1', '2']}, :consistent_read => true)
|
49
|
+
#
|
50
|
+
# @param [Hash] table_ids the hash of tables and IDs to retrieve
|
51
|
+
# @param [Hash] options to be passed to underlying BatchGet call
|
52
|
+
#
|
53
|
+
# @return [Hash] a hash where keys are the table names and the values are the retrieved items
|
54
|
+
#
|
55
|
+
# @since 0.2.0
|
56
|
+
def batch_get_item(table_ids, options = {})
|
57
|
+
binding.pry
|
58
|
+
connection.batch_get do |batch|
|
59
|
+
table_ids.each do |table, keys|
|
60
|
+
batch.table(table, :all, keys)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
|
66
|
+
#return hash if table_ids.all?{|k, v| v.empty?}
|
67
|
+
#table_ids.each do |t, ids|
|
68
|
+
# Array(ids).in_groups_of(100, false) do |group|
|
69
|
+
# batch = AWS::DynamoDB::BatchGet.new(:config => @@connection.config)
|
70
|
+
# batch.table(t, :all, Array(group), options) unless group.nil? || group.empty?
|
71
|
+
# batch.each do |table_name, attributes|
|
72
|
+
# hash[table_name] << attributes.symbolize_keys!
|
73
|
+
# end
|
74
|
+
# end
|
75
|
+
#end
|
76
|
+
#hash
|
77
|
+
end
|
78
|
+
|
79
|
+
# Delete many items at once from DynamoDB. More efficient than delete each item individually.
|
80
|
+
#
|
81
|
+
# @example Delete IDs 1 and 2 from the table testtable
|
82
|
+
# Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => ['1', '2'])
|
83
|
+
#or
|
84
|
+
# Dynamoid::Adapter::AwsSdk.batch_delete_item('table1' => [['hk1', 'rk2'], ['hk1', 'rk2']]]))
|
85
|
+
#
|
86
|
+
# @param [Hash] options the hash of tables and IDs to delete
|
87
|
+
#
|
88
|
+
# @return nil
|
89
|
+
#
|
90
|
+
def batch_delete_item(options)
|
91
|
+
return nil if options.all?{|k, v| v.empty?}
|
92
|
+
options.each do |t, ids|
|
93
|
+
Array(ids).in_groups_of(25, false) do |group|
|
94
|
+
batch = AWS::DynamoDB::BatchWrite.new(:config => @@connection.config)
|
95
|
+
batch.delete(t,group)
|
96
|
+
batch.process!
|
97
|
+
end
|
98
|
+
end
|
99
|
+
nil
|
100
|
+
end
|
101
|
+
|
102
|
+
# Create a table on DynamoDB. This usually takes a long time to complete.
|
103
|
+
#
|
104
|
+
# @param [String] table_name the name of the table to create
|
105
|
+
# @param [Symbol] key the table's primary key (defaults to :id)
|
106
|
+
# @param [Hash] options provide a range_key here if you want one for the table
|
107
|
+
#
|
108
|
+
# @since 0.2.0
|
109
|
+
def create_table(table_name, key = :id, options = {})
|
110
|
+
Dynamoid.logger.info "Creating #{table_name} table. This could take a while."
|
111
|
+
options[:hash_key] ||= {key.to_sym => :string}
|
112
|
+
read_capacity = options[:read_capacity] || Dynamoid::Config.read_capacity
|
113
|
+
write_capacity = options[:write_capacity] || Dynamoid::Config.write_capacity
|
114
|
+
table = @@connection.tables.create(table_name, read_capacity, write_capacity, options)
|
115
|
+
sleep 0.5 while table.status == :creating
|
116
|
+
return table
|
117
|
+
end
|
118
|
+
|
119
|
+
# Removes an item from DynamoDB.
|
120
|
+
#
|
121
|
+
# @param [String] table_name the name of the table
|
122
|
+
# @param [String] key the hash key of the item to delete
|
123
|
+
# @param [Number] range_key the range key of the item to delete, required if the table has a composite key
|
124
|
+
#
|
125
|
+
# @since 0.2.0
|
126
|
+
def delete_item(table_name, key, options = {})
|
127
|
+
range_key = options.delete(:range_key)
|
128
|
+
table = get_table(table_name)
|
129
|
+
result = table.items.at(key, range_key)
|
130
|
+
result.delete unless result.attributes.to_h.empty?
|
131
|
+
true
|
132
|
+
end
|
133
|
+
|
134
|
+
# Deletes an entire table from DynamoDB. Only 10 tables can be in the deleting state at once,
|
135
|
+
# so if you have more this method may raise an exception.
|
136
|
+
#
|
137
|
+
# @param [String] table_name the name of the table to destroy
|
138
|
+
#
|
139
|
+
# @since 0.2.0
|
140
|
+
def delete_table(table_name)
|
141
|
+
Dynamoid.logger.info "Deleting #{table_name} table. This could take a while."
|
142
|
+
table = @@connection.tables[table_name]
|
143
|
+
table.delete
|
144
|
+
sleep 0.5 while table.exists? == true
|
145
|
+
end
|
146
|
+
|
147
|
+
# @todo Add a DescribeTable method.
|
148
|
+
|
149
|
+
# Fetches an item from DynamoDB.
|
150
|
+
#
|
151
|
+
# @param [String] table_name the name of the table
|
152
|
+
# @param [String] key the hash key of the item to find
|
153
|
+
# @param [Number] range_key the range key of the item to find, required if the table has a composite key
|
154
|
+
#
|
155
|
+
# @return [Hash] a hash representing the raw item in DynamoDB
|
156
|
+
#
|
157
|
+
# @since 0.2.0
|
158
|
+
def get_item(table_name, key, options = {})
|
159
|
+
range_key = options.delete(:range_key)
|
160
|
+
table = get_table(table_name)
|
161
|
+
|
162
|
+
result = table.items.at(key, range_key).attributes.to_h(options)
|
163
|
+
|
164
|
+
if result.empty?
|
165
|
+
nil
|
166
|
+
else
|
167
|
+
result.symbolize_keys!
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def update_item(table_name, key, options = {}, &block)
|
172
|
+
range_key = options.delete(:range_key)
|
173
|
+
conditions = options.delete(:conditions) || {}
|
174
|
+
table = get_table(table_name)
|
175
|
+
item = table.items.at(key, range_key)
|
176
|
+
item.attributes.update(conditions.merge(:return => :all_new), &block)
|
177
|
+
rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException
|
178
|
+
raise Dynamoid::Errors::ConditionalCheckFailedException
|
179
|
+
end
|
180
|
+
|
181
|
+
# List all tables on DynamoDB.
|
182
|
+
#
|
183
|
+
# @since 0.2.0
|
184
|
+
def list_tables
|
185
|
+
connection.db.tables.collect(&:name)
|
186
|
+
end
|
187
|
+
|
188
|
+
# Persists an item on DynamoDB.
|
189
|
+
#
|
190
|
+
# @param [String] table_name the name of the table
|
191
|
+
# @param [Object] object a hash or Dynamoid object to persist
|
192
|
+
#
|
193
|
+
# @since 0.2.0
|
194
|
+
def put_item(table_name, object, options = nil)
|
195
|
+
connection.put_item(table_name, object, options)
|
196
|
+
end
|
197
|
+
|
198
|
+
# Query the DynamoDB table. This employs DynamoDB's indexes so is generally faster than scanning, but is
|
199
|
+
# only really useful for range queries, since it can only find by one hash key at once. Only provide
|
200
|
+
# one range key to the hash.
|
201
|
+
#
|
202
|
+
# @param [String] table_name the name of the table
|
203
|
+
# @param [Hash] opts the options to query the table with
|
204
|
+
# @option opts [String] :hash_value the value of the hash key to find
|
205
|
+
# @option opts [Range] :range_value find the range key within this range
|
206
|
+
# @option opts [Number] :range_greater_than find range keys greater than this
|
207
|
+
# @option opts [Number] :range_less_than find range keys less than this
|
208
|
+
# @option opts [Number] :range_gte find range keys greater than or equal to this
|
209
|
+
# @option opts [Number] :range_lte find range keys less than or equal to this
|
210
|
+
#
|
211
|
+
# @return [Enumerator] an iterator of all matching items
|
212
|
+
#
|
213
|
+
# @since 0.2.0
|
214
|
+
def query(table_name, opts = {})
|
215
|
+
table = get_table(table_name)
|
216
|
+
|
217
|
+
Enumerator.new do |yielder|
|
218
|
+
consistent_opts = { :consistent_read => opts[:consistent_read] || false }
|
219
|
+
if table.composite_key?
|
220
|
+
table.items.query(opts).each do |data|
|
221
|
+
if opts.has_key? :select
|
222
|
+
yielder.yield data.attributes.symbolize_keys!
|
223
|
+
else
|
224
|
+
yielder.yield data.attributes.to_h(consistent_opts).symbolize_keys!
|
225
|
+
end
|
226
|
+
end
|
227
|
+
else
|
228
|
+
yielder.yield get_item(table_name, opts[:hash_value])
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
# Scan the DynamoDB table. This is usually a very slow operation as it naively filters all data on
|
234
|
+
# the DynamoDB servers.
|
235
|
+
#
|
236
|
+
# @param [String] table_name the name of the table
|
237
|
+
# @param [Hash] scan_hash a hash of attributes: matching records will be returned by the scan
|
238
|
+
#
|
239
|
+
# @return [Enumerator] an iterator of all matching items
|
240
|
+
#
|
241
|
+
# @since 0.2.0
|
242
|
+
def scan(table_name, scan_hash, select_opts)
|
243
|
+
table = get_table(table_name)
|
244
|
+
Enumerator.new do |yielder|
|
245
|
+
table.items.where(scan_hash).select(select_opts).each do |data|
|
246
|
+
yielder.yield data.attributes.symbolize_keys!
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
# @todo Add an UpdateItem method.
|
252
|
+
|
253
|
+
# @todo Add an UpdateTable method.
|
254
|
+
|
255
|
+
def get_table(table_name)
|
256
|
+
unless table = table_cache[table_name]
|
257
|
+
table = @@connection.db.tables[table_name]
|
258
|
+
table.load_schema
|
259
|
+
table_cache[table_name] = table
|
260
|
+
end
|
261
|
+
table
|
262
|
+
end
|
263
|
+
|
264
|
+
def table_cache
|
265
|
+
@table_cache ||= {}
|
266
|
+
end
|
267
|
+
|
268
|
+
# Number of items from a table
|
269
|
+
#
|
270
|
+
# @param [String] table_name the name of the table
|
271
|
+
#
|
272
|
+
# @return [Integer] the number of items from a table
|
273
|
+
#
|
274
|
+
# @since 0.6.1
|
275
|
+
def count(table_name)
|
276
|
+
table = get_table(table_name)
|
277
|
+
table.items.count
|
278
|
+
end
|
279
|
+
end
|
280
|
+
end
|
281
|
+
end
|
data/lib/ebx_deliver.rb
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
module EbxDeliver
|
2
|
+
class AwsSnsWriter
|
3
|
+
attr_accessor :db
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
# TODO fix
|
7
|
+
@topic = AWS.sns.topics.create('development-sns')
|
8
|
+
|
9
|
+
@db = AWS::DynamoDB.new
|
10
|
+
end
|
11
|
+
|
12
|
+
def batch_get_item(&block)
|
13
|
+
@db.batch_get_item(&block)
|
14
|
+
end
|
15
|
+
|
16
|
+
def batch_write_item(options)
|
17
|
+
@db.client.batch_write_item(options)
|
18
|
+
@topic.publish({
|
19
|
+
method: 'batch_write_item',
|
20
|
+
args: [options]
|
21
|
+
}.to_json)
|
22
|
+
end
|
23
|
+
|
24
|
+
def batch_delete_item(options)
|
25
|
+
@topic.publish({
|
26
|
+
method: 'batch_delete_item',
|
27
|
+
args: [options]
|
28
|
+
}.to_json)
|
29
|
+
end
|
30
|
+
|
31
|
+
def create_table(options = {})
|
32
|
+
@topic.publish({
|
33
|
+
method: 'create_table',
|
34
|
+
args: [options]
|
35
|
+
}.to_json)
|
36
|
+
end
|
37
|
+
|
38
|
+
def delete_item(options = {})
|
39
|
+
@topic.publish({
|
40
|
+
method: 'delete_item',
|
41
|
+
args: [options]
|
42
|
+
}.to_json)
|
43
|
+
end
|
44
|
+
|
45
|
+
def delete_table(options = {})
|
46
|
+
@topic.publish({
|
47
|
+
method: 'delete_table',
|
48
|
+
args: [table_name]
|
49
|
+
}.to_json)
|
50
|
+
end
|
51
|
+
|
52
|
+
def describe_table(options = {})
|
53
|
+
@db.client.describe_table(options)
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_item(table_name, key, options = {})
|
57
|
+
@db.client.get_item(options)
|
58
|
+
end
|
59
|
+
|
60
|
+
def list_tables
|
61
|
+
@db.client.list_tables(options)
|
62
|
+
end
|
63
|
+
|
64
|
+
def put_item(table_name, object, options = nil)
|
65
|
+
@topic.publish({
|
66
|
+
method: 'put_item',
|
67
|
+
args: [table_name, object, options]
|
68
|
+
}.to_json)
|
69
|
+
end
|
70
|
+
|
71
|
+
def query(table_name, opts = {})
|
72
|
+
@db.client.query(options)
|
73
|
+
end
|
74
|
+
|
75
|
+
def scan(options)
|
76
|
+
@db.client.scan(options)
|
77
|
+
end
|
78
|
+
|
79
|
+
def update_item(options = {})
|
80
|
+
@topic.publish({
|
81
|
+
method: 'update_item',
|
82
|
+
args: [options]
|
83
|
+
}.to_json)
|
84
|
+
end
|
85
|
+
|
86
|
+
def update_table(options = {})
|
87
|
+
@topic.publish({
|
88
|
+
method: 'update_item',
|
89
|
+
args: [options]
|
90
|
+
}.to_json)
|
91
|
+
end
|
92
|
+
|
93
|
+
def self.send_test
|
94
|
+
w = AwsSnsWriter.new
|
95
|
+
w.connect!
|
96
|
+
|
97
|
+
w.create_table('testtt')
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module EbxDeliver
|
2
|
+
class AwsSqsReader
|
3
|
+
def connect!
|
4
|
+
# TODO pull name from settings
|
5
|
+
@queue = sqs.queues.named('development-sns')
|
6
|
+
|
7
|
+
@db = AWS::DynamoDB.new
|
8
|
+
|
9
|
+
@queue.poll do |notification|
|
10
|
+
msg = JSON.parse(JSON.parse(notification.body)['Message'])
|
11
|
+
Rails.logger.info "MSG Received #{msg}"
|
12
|
+
send(msg['method'], *msg['args'])
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def create_table(options)
|
17
|
+
@db.client.create_table(options)
|
18
|
+
end
|
19
|
+
|
20
|
+
def batch_delete_item(options)
|
21
|
+
@db.client.batch_delete_item(options)
|
22
|
+
end
|
23
|
+
|
24
|
+
def create_table(options = {})
|
25
|
+
@db.client.create_table(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def delete_item(options = {})
|
29
|
+
@db.client.delete_item(options)
|
30
|
+
end
|
31
|
+
|
32
|
+
def delete_table(options = {})
|
33
|
+
@db.client.delete_table(options)
|
34
|
+
end
|
35
|
+
|
36
|
+
def put_item(table_name, object, options = {})
|
37
|
+
table = @db.tables[table_name]
|
38
|
+
table.load_schema
|
39
|
+
table.items.create(
|
40
|
+
object.delete_if{|k, v| v.nil? || (v.respond_to?(:empty?) && v.empty?)},
|
41
|
+
options || {}
|
42
|
+
)
|
43
|
+
rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
|
44
|
+
raise Dynamoid::Errors::ConditionalCheckFailedException
|
45
|
+
end
|
46
|
+
|
47
|
+
def update_item(options = {})
|
48
|
+
@db.client.update_item(options)
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_table(options = {})
|
52
|
+
@db.client.update_table(options)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
metadata
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ebx_deliver
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Alex Bullard
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-10-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.3'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.3'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: dynamoid
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.7.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.7.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: aws-sdk
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.17.0
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.17.0
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ! '>'
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ! '>'
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: eb eXtended cross region db command distributor
|
84
|
+
email:
|
85
|
+
- abullrd@gmail.com
|
86
|
+
executables: []
|
87
|
+
extensions: []
|
88
|
+
extra_rdoc_files: []
|
89
|
+
files:
|
90
|
+
- .gitignore
|
91
|
+
- .ruby-version
|
92
|
+
- Gemfile
|
93
|
+
- Gemfile.lock
|
94
|
+
- LICENSE
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- ebx_deliver.gemspec
|
98
|
+
- lib/dynamoid/adapter/aws_multi_region_adapter.rb
|
99
|
+
- lib/ebx_deliver.rb
|
100
|
+
- lib/ebx_deliver/aws_sns_writer.rb
|
101
|
+
- lib/ebx_deliver/aws_sqs_reader.rb
|
102
|
+
- lib/ebx_deliver/version.rb
|
103
|
+
homepage: https://github.com/Funnerator/ebx_deliver.git
|
104
|
+
licenses:
|
105
|
+
- MIT
|
106
|
+
metadata: {}
|
107
|
+
post_install_message:
|
108
|
+
rdoc_options: []
|
109
|
+
require_paths:
|
110
|
+
- lib
|
111
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - ! '>='
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
116
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - ! '>='
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
121
|
+
requirements: []
|
122
|
+
rubyforge_project:
|
123
|
+
rubygems_version: 2.0.6
|
124
|
+
signing_key:
|
125
|
+
specification_version: 4
|
126
|
+
summary: Distributes writes to Amazon's DynamoDB through SNS to allow for multiple
|
127
|
+
cross region databases
|
128
|
+
test_files: []
|
129
|
+
has_rdoc:
|