export_mongo_s3 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +24 -0
- data/Gemfile +3 -0
- data/README.md +20 -0
- data/bin/export_mongo_s3 +11 -0
- data/export_mongo_s3.gemspec +19 -0
- data/lib/export_mongo_s3.rb +28 -0
- data/lib/export_mongo_s3/application.rb +226 -0
- data/lib/export_mongo_s3/db.rb +142 -0
- data/lib/export_mongo_s3/scheduler.rb +97 -0
- data/lib/export_mongo_s3/storage.rb +76 -0
- data/lib/helpers/config.yml.template +36 -0
- data/lib/helpers/fixnum.rb +11 -0
- data/lib/helpers/hash.rb +26 -0
- data/lib/helpers/time.rb +25 -0
- metadata +88 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d3c8c2fbe8679bb4b3c17ea0ceed2c86492a00c6
|
4
|
+
data.tar.gz: 7e2192e16878e2cd87a4f6e4fa247ca081e60284
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d9f0bf7dcbd0500bddfbc24c7e8ed89e0f3ba034a582a715da5145a999c61410a98ff58edc03adb18c43e6ad45a72918425097c898a362603d9164281138b774
|
7
|
+
data.tar.gz: 2fba509d5f4ba909c62437e95914586a6fa812f8ab92a497d71dbd4ef4d8a45da1df562b8e43c48e6f9f28b6b58a734f5ec7a5fa470cc5818a0297e4ddf37f83
|
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
*.idea
|
4
|
+
.bundle
|
5
|
+
.config
|
6
|
+
.yardoc
|
7
|
+
Gemfile.lock
|
8
|
+
InstalledFiles
|
9
|
+
_yardoc
|
10
|
+
coverage
|
11
|
+
doc/
|
12
|
+
lib/bundler/man
|
13
|
+
pkg
|
14
|
+
rdoc
|
15
|
+
spec/reports
|
16
|
+
test/tmp
|
17
|
+
test/version_tmp
|
18
|
+
tmp
|
19
|
+
*.bundle
|
20
|
+
*.so
|
21
|
+
*.o
|
22
|
+
*.a
|
23
|
+
mkmf.log
|
24
|
+
*.yml
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# ExportMongoS3
|
2
|
+
|
3
|
+
Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'export_mongo_s3'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install export_mongo_s3
|
18
|
+
|
19
|
+
## Help
|
20
|
+
|
data/bin/export_mongo_s3
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Gem::Specification.new do |spec|
|
2
|
+
spec.name = 'export_mongo_s3'
|
3
|
+
spec.version = '0.0.1'
|
4
|
+
spec.authors = ['Yakupov Dima']
|
5
|
+
spec.email = ['yakupov.dima@mail.ru']
|
6
|
+
spec.summary = "Some summary"
|
7
|
+
spec.description = "Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3"
|
8
|
+
spec.homepage = ''
|
9
|
+
spec.license = 'MIT'
|
10
|
+
|
11
|
+
spec.files = `git ls-files -z`.split("\x0")
|
12
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
14
|
+
spec.require_paths = ["lib"]
|
15
|
+
|
16
|
+
spec.add_development_dependency 'bundler', '~> 1.6'
|
17
|
+
|
18
|
+
spec.add_runtime_dependency 'aws-sdk', '~> 1.57'
|
19
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'aws-sdk'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'digest/md5'
|
5
|
+
require 'tmpdir'
|
6
|
+
|
7
|
+
require_relative 'export_mongo_s3/application'
|
8
|
+
require_relative 'export_mongo_s3/db'
|
9
|
+
require_relative 'export_mongo_s3/storage'
|
10
|
+
require_relative 'export_mongo_s3/scheduler'
|
11
|
+
|
12
|
+
require_relative 'helpers/fixnum'
|
13
|
+
require_relative 'helpers/hash'
|
14
|
+
require_relative 'helpers/time'
|
15
|
+
|
16
|
+
module ExportMongoS3
|
17
|
+
|
18
|
+
def self.name
|
19
|
+
File.basename( __FILE__, '.rb')
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.root_path
|
23
|
+
@root_path if @root_path
|
24
|
+
spec = Gem::Specification.find_by_name(self.name)
|
25
|
+
@root_path = spec.gem_dir
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,226 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Application
|
3
|
+
|
4
|
+
def initialize(argv)
|
5
|
+
@parser = OptionParser.new
|
6
|
+
|
7
|
+
@params = parse_options(argv)
|
8
|
+
@config = parse_config(@params[:config])
|
9
|
+
|
10
|
+
db_options = @config[:mongo]
|
11
|
+
db_options.merge!(collections: @config[:export][:collections])
|
12
|
+
|
13
|
+
@db = Db.new(db_options)
|
14
|
+
|
15
|
+
@storage = Storage.new(@config[:s3])
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
public
|
20
|
+
|
21
|
+
def run
|
22
|
+
|
23
|
+
case
|
24
|
+
when @params[:export_list] # EXPORT_LIST
|
25
|
+
show_uploaded_files(@params[:export_list])
|
26
|
+
|
27
|
+
when @params[:export_all] # EXPORT_ALL
|
28
|
+
dbs_str = @config[:export][:dbs]
|
29
|
+
|
30
|
+
if dbs_str.nil? || dbs_str.empty?
|
31
|
+
raise 'config.yml::export.dbs is empty'
|
32
|
+
end
|
33
|
+
|
34
|
+
db_names = dbs_str.split(',').each { |db_name| db_name.strip! }
|
35
|
+
|
36
|
+
export(db_names)
|
37
|
+
|
38
|
+
when @params[:export] # EXPORT
|
39
|
+
export([@params[:export]])
|
40
|
+
|
41
|
+
when @params[:cron_update] || @params[:cron_clear] # CRON
|
42
|
+
cron_options =
|
43
|
+
{
|
44
|
+
update: @params[:cron_update],
|
45
|
+
clear: @params[:cron_clear],
|
46
|
+
config: @params[:config],
|
47
|
+
time: @config[:backup][:cron_time],
|
48
|
+
}
|
49
|
+
|
50
|
+
Scheduler.new(cron_options).execute
|
51
|
+
|
52
|
+
else
|
53
|
+
puts "\n#{@parser}\n"
|
54
|
+
exit(0)
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def show_uploaded_files(db_name)
|
63
|
+
|
64
|
+
files = @storage.get_uploaded_files(db_name)
|
65
|
+
|
66
|
+
if files.empty?
|
67
|
+
puts 'Files not found'
|
68
|
+
else
|
69
|
+
|
70
|
+
puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')
|
71
|
+
|
72
|
+
files.each do |file|
|
73
|
+
|
74
|
+
file_name = file.key
|
75
|
+
file_size = file.content_length / 1024 / 1024
|
76
|
+
file_last_modified = file.last_modified
|
77
|
+
|
78
|
+
puts sprintf('%-30s %-15s %-20s', file_name, file_size, file_last_modified)
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def export(db_names)
|
85
|
+
|
86
|
+
db_names.each do |db_name|
|
87
|
+
|
88
|
+
puts "export db #{db_name.upcase}:"
|
89
|
+
|
90
|
+
tmp_dir = get_temp_dir
|
91
|
+
|
92
|
+
begin
|
93
|
+
|
94
|
+
export_path = File.join(tmp_dir, db_name)
|
95
|
+
|
96
|
+
puts "\t make csv..."
|
97
|
+
@db.export_db(db_name, export_path)
|
98
|
+
|
99
|
+
if Dir["#{export_path}/*"].empty?
|
100
|
+
puts "\t [skip] db is empty"
|
101
|
+
|
102
|
+
else
|
103
|
+
|
104
|
+
puts "\t compress..."
|
105
|
+
zip_result = %x(zip -6 -r '#{export_path}.zip' '#{export_path}/' -j)
|
106
|
+
raise "Error zip. Msg: #{zip_result}" unless $?.exitstatus.zero?
|
107
|
+
|
108
|
+
FileUtils.rm_rf(export_path)
|
109
|
+
|
110
|
+
zip_file_path = "#{export_path}.zip"
|
111
|
+
|
112
|
+
if File.exists?(zip_file_path)
|
113
|
+
puts "\t upload to s3..."
|
114
|
+
@storage.upload(db_name, zip_file_path)
|
115
|
+
|
116
|
+
File.delete(zip_file_path)
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
|
121
|
+
ensure
|
122
|
+
FileUtils.remove_entry_secure(tmp_dir)
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
puts '[done] export'
|
128
|
+
end
|
129
|
+
|
130
|
+
def create_config(path)
|
131
|
+
|
132
|
+
path = '.' if path.nil? || path == ''
|
133
|
+
|
134
|
+
file = File.join(path, 'config.yml')
|
135
|
+
|
136
|
+
if File.exists?(file)
|
137
|
+
raise "create_config: '#{file}' already exists"
|
138
|
+
elsif File.exists?(file.downcase)
|
139
|
+
raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
|
140
|
+
elsif !File.exists?(File.dirname(file))
|
141
|
+
raise "create_config: directory '#{File.dirname(file)}' does not exist"
|
142
|
+
else
|
143
|
+
file_template = File.join(ExportMongoS3.root_path, 'lib/helpers/config.yml.template')
|
144
|
+
|
145
|
+
FileUtils.cp file_template, file
|
146
|
+
end
|
147
|
+
|
148
|
+
puts "[done] file #{file} was created"
|
149
|
+
end
|
150
|
+
|
151
|
+
def parse_options(argv)
|
152
|
+
params = {}
|
153
|
+
|
154
|
+
@parser.on('--export_all', 'Export databases specified in config.yml and upload to S3 bucket') do
|
155
|
+
params[:export_all] = true
|
156
|
+
end
|
157
|
+
@parser.on('--export DB_NAME', String, 'Export database and upload to S3 bucket') do |db_name|
|
158
|
+
params[:export] = db_name
|
159
|
+
end
|
160
|
+
@parser.on('-l', '--list_exported [DB_NAME]', String, 'Show list of exported dbs') do |db_name|
|
161
|
+
params[:export_list] = db_name || ''
|
162
|
+
end
|
163
|
+
@parser.on('--write_cron', 'Add/update export_all job') do
|
164
|
+
params[:cron_update] = true
|
165
|
+
end
|
166
|
+
@parser.on('--clear_cron', 'Clear export_all job') do
|
167
|
+
params[:cron_clear] = true
|
168
|
+
end
|
169
|
+
@parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default: ./config.yml') do |path|
|
170
|
+
params[:config] = path || ''
|
171
|
+
end
|
172
|
+
@parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
|
173
|
+
create_config(path)
|
174
|
+
exit(0)
|
175
|
+
end
|
176
|
+
@parser.on('-h', '--help', 'Show help') do
|
177
|
+
puts "\n#{@parser}\n"
|
178
|
+
exit(0)
|
179
|
+
end
|
180
|
+
|
181
|
+
@parser.parse!(argv)
|
182
|
+
|
183
|
+
if [params[:export_all], params[:export], params[:export_list], params[:cron_update], params[:cron_clear]].compact.length > 1
|
184
|
+
raise 'Can only export_all, export, list_exported, write_cron or clear_cron. Choose one.'
|
185
|
+
end
|
186
|
+
|
187
|
+
if params[:config].nil? || params[:config] == ''
|
188
|
+
params[:config] = './config.yml'
|
189
|
+
end
|
190
|
+
|
191
|
+
params[:config] = File.absolute_path(params[:config])
|
192
|
+
|
193
|
+
params
|
194
|
+
end
|
195
|
+
|
196
|
+
def parse_config(config_path)
|
197
|
+
|
198
|
+
begin
|
199
|
+
config = YAML.load(File.read(config_path))
|
200
|
+
rescue Errno::ENOENT
|
201
|
+
raise "Could not find config file '#{config_path}'"
|
202
|
+
rescue ArgumentError => error
|
203
|
+
raise "Could not parse config file '#{config_path}' - #{error}"
|
204
|
+
end
|
205
|
+
|
206
|
+
config.deep_symbolize_keys!
|
207
|
+
|
208
|
+
raise 'config.yml. Section <export> not found' if config[:export].nil?
|
209
|
+
raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
|
210
|
+
raise 'config.yml. Section <s3> not found' if config[:s3].nil?
|
211
|
+
|
212
|
+
config
|
213
|
+
end
|
214
|
+
|
215
|
+
def get_temp_dir
|
216
|
+
temp_dir = @config[:export][:temp_directory]
|
217
|
+
|
218
|
+
if temp_dir.nil? || temp_dir == ''
|
219
|
+
temp_dir = Dir.tmpdir
|
220
|
+
end
|
221
|
+
|
222
|
+
Dir.mktmpdir(nil, temp_dir)
|
223
|
+
end
|
224
|
+
|
225
|
+
end
|
226
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Db
|
3
|
+
|
4
|
+
SYSTEM_COLLECTIONS = %w(admin_users fs.chunks fs.files system.indexes)
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@options = options
|
8
|
+
@connection_options = connection(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
public
|
13
|
+
|
14
|
+
def export_db(db, out_path)
|
15
|
+
|
16
|
+
collection_settings_map = prepared_collection_settings
|
17
|
+
|
18
|
+
if collection_settings_map.empty?
|
19
|
+
collection_names = get_collections(db)
|
20
|
+
else
|
21
|
+
collection_names = collection_settings_map.keys
|
22
|
+
end
|
23
|
+
|
24
|
+
collection_names.each do |collection_name|
|
25
|
+
|
26
|
+
collection_settings = collection_settings_map[collection_name]
|
27
|
+
|
28
|
+
if collection_settings.nil?
|
29
|
+
export(db, collection_name, out_path)
|
30
|
+
else
|
31
|
+
fields = collection_settings[:fields]
|
32
|
+
query = collection_settings[:query]
|
33
|
+
|
34
|
+
export(db, collection_name, out_path, fields, query)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def connection(options)
|
43
|
+
host = (options[:host].nil? || options[:host] == '') ? 'localhost' : options[:host]
|
44
|
+
port = options[:port].nil? ? 27017 : options[:port]
|
45
|
+
username = options[:username]
|
46
|
+
password = options[:password]
|
47
|
+
authentication_database = options[:authentication_database]
|
48
|
+
|
49
|
+
auth_options = ''
|
50
|
+
|
51
|
+
unless username.nil? || username == '' || password.nil? || password == ''
|
52
|
+
auth_options = "-u '#{username}' -p '#{password}'"
|
53
|
+
auth_options << " --authenticationDatabase '#{authentication_database}'" unless authentication_database.nil? || authentication_database == ''
|
54
|
+
end
|
55
|
+
|
56
|
+
"--host '#{host}' --port '#{port}' #{auth_options}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def export(db, collection, out_path, fields = [], query = nil)
|
60
|
+
|
61
|
+
if fields.empty?
|
62
|
+
fields = get_fields(db, collection)
|
63
|
+
end
|
64
|
+
|
65
|
+
return if fields.empty?
|
66
|
+
|
67
|
+
command = 'mongoexport'
|
68
|
+
command << " #{@connection_options}"
|
69
|
+
command << ' --csv'
|
70
|
+
command << " --db '#{db}'"
|
71
|
+
command << " --collection '#{collection}'"
|
72
|
+
command << " --fields '#{fields.join(',')}'"
|
73
|
+
command << " --query '#{query}'" unless query.nil?
|
74
|
+
command << " --out '#{out_path}/#{collection}.csv'"
|
75
|
+
command << ' > /dev/null'
|
76
|
+
|
77
|
+
system(command)
|
78
|
+
raise "Error mongoexport '#{db}'" unless $?.exitstatus.zero?
|
79
|
+
end
|
80
|
+
|
81
|
+
def get_collections(db)
|
82
|
+
command = 'mongo'
|
83
|
+
command << " #{@connection_options}"
|
84
|
+
command << " #{db} --eval 'rs.slaveOk(); db.getCollectionNames();'"
|
85
|
+
command << ' --quiet'
|
86
|
+
|
87
|
+
result = %x(#{command})
|
88
|
+
raise "Error get collections for db '#{db}'. Msg: #{result}" unless $?.exitstatus.zero?
|
89
|
+
|
90
|
+
result.strip.split(',') - SYSTEM_COLLECTIONS
|
91
|
+
end
|
92
|
+
|
93
|
+
def get_fields(db, collection)
|
94
|
+
command = 'mongo'
|
95
|
+
command << " #{@connection_options}"
|
96
|
+
command << " #{db} --eval 'rs.slaveOk(); var fields = []; for(var field in db.#{collection}.find().sort({_id: -1}).limit(1)[0]) { fields.push(field); }; fields;'"
|
97
|
+
command << ' --quiet'
|
98
|
+
|
99
|
+
result = %x(#{command})
|
100
|
+
raise "Error get fields for db '#{db}' and collection '#{collection}'. Msg: #{result}" unless $?.exitstatus.zero?
|
101
|
+
|
102
|
+
result.strip.split(',')
|
103
|
+
end
|
104
|
+
|
105
|
+
def prepared_collection_settings
|
106
|
+
@prepared_collection_settings if @prepared_collection_settings
|
107
|
+
|
108
|
+
collection_settings = @options[:collections]
|
109
|
+
|
110
|
+
prepared_settings = {}
|
111
|
+
|
112
|
+
if collection_settings.is_a?(Array)
|
113
|
+
collection_settings.each do |collection|
|
114
|
+
|
115
|
+
name = collection['name']
|
116
|
+
query = collection['query'].nil? || collection['query'] == '' ? nil : collection['query'].to_s
|
117
|
+
|
118
|
+
fields = if collection['fields'].is_a?(String)
|
119
|
+
collection['fields'].split(',').each { |field| field.strip! }
|
120
|
+
else
|
121
|
+
[]
|
122
|
+
end
|
123
|
+
|
124
|
+
|
125
|
+
if name.nil? || name == ''
|
126
|
+
raise "Not valid param <name: #{collection}>"
|
127
|
+
end
|
128
|
+
|
129
|
+
prepared_settings[name] =
|
130
|
+
{
|
131
|
+
name: name,
|
132
|
+
fields: fields,
|
133
|
+
query: query
|
134
|
+
}
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
@prepared_collection_settings = prepared_settings
|
139
|
+
end
|
140
|
+
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Scheduler
|
3
|
+
|
4
|
+
def initialize(options = {})
|
5
|
+
@options = options
|
6
|
+
|
7
|
+
if [@options[:write], @options[:clear]].compact.length > 1
|
8
|
+
raise 'cron: Can only write or clear. Choose one.'
|
9
|
+
end
|
10
|
+
|
11
|
+
unless @options[:time] =~ /\A[*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-6]+\z/
|
12
|
+
raise 'config.yml: cron_time is not valid'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute
|
17
|
+
write_crontab(updated_crontab)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def read_crontab
|
23
|
+
return @read_crontab if @read_crontab
|
24
|
+
|
25
|
+
command = 'crontab -l'
|
26
|
+
|
27
|
+
command_results = %x[#{command} 2> /dev/null]
|
28
|
+
|
29
|
+
@read_crontab = $?.exitstatus.zero? ? prepare(command_results) : ''
|
30
|
+
end
|
31
|
+
|
32
|
+
def prepare(contents)
|
33
|
+
# Some cron implementations require all non-comment lines to be newline-
|
34
|
+
# terminated. (issue #95) Strip all newlines and replace with the default
|
35
|
+
# platform record seperator ($/)
|
36
|
+
contents.gsub!(/\s+$/, $/)
|
37
|
+
end
|
38
|
+
|
39
|
+
def write_crontab(contents)
|
40
|
+
command = 'crontab -'
|
41
|
+
|
42
|
+
IO.popen(command, 'r+') do |crontab|
|
43
|
+
crontab.write(contents)
|
44
|
+
crontab.close_write
|
45
|
+
end
|
46
|
+
|
47
|
+
success = $?.exitstatus.zero?
|
48
|
+
|
49
|
+
if success
|
50
|
+
action = @options[:update] ? 'updated' : 'cleared'
|
51
|
+
puts "[done] crontab file #{action}"
|
52
|
+
exit(0)
|
53
|
+
else
|
54
|
+
raise "Couldn't write crontab"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def updated_crontab
|
59
|
+
# Check for unopened or unclosed identifier blocks
|
60
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && (read_crontab =~ Regexp.new("^#{comment_close}\s*$")).nil?
|
61
|
+
raise "Unclosed indentifier; Your crontab file contains '#{comment_open}', but no '#{comment_close}'"
|
62
|
+
elsif (read_crontab =~ Regexp.new("^#{comment_open}\s*$")).nil? && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
63
|
+
raise "Unopened indentifier; Your crontab file contains '#{comment_close}', but no '#{comment_open}'"
|
64
|
+
end
|
65
|
+
|
66
|
+
# If an existing identier block is found, replace it with the new cron entries
|
67
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
68
|
+
# If the existing crontab file contains backslashes they get lost going through gsub.
|
69
|
+
# .gsub('\\', '\\\\\\') preserves them. Go figure.
|
70
|
+
read_crontab.gsub(Regexp.new("^#{comment_open}\s*$.+^#{comment_close}\s*$", Regexp::MULTILINE), crontab_task.chomp.gsub('\\', '\\\\\\'))
|
71
|
+
else # Otherwise, append the new cron entries after any existing ones
|
72
|
+
[read_crontab, crontab_task].join("\n\n")
|
73
|
+
end.gsub(/\n{3,}/, "\n\n") # More than two newlines becomes just two.
|
74
|
+
end
|
75
|
+
|
76
|
+
def crontab_task
|
77
|
+
return '' if @options[:clear]
|
78
|
+
[comment_open, crontab_job, comment_close].compact.join("\n") + "\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
def crontab_job
|
82
|
+
"#{@options[:time]} /bin/bash -l -c '#{ExportMongoS3.name} --export_all --config #{@options[:config]}'"
|
83
|
+
end
|
84
|
+
|
85
|
+
def comment_base
|
86
|
+
"#{ExportMongoS3.name} task"
|
87
|
+
end
|
88
|
+
|
89
|
+
def comment_open
|
90
|
+
"# Begin #{comment_base}"
|
91
|
+
end
|
92
|
+
|
93
|
+
def comment_close
|
94
|
+
"# End #{comment_base}"
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Storage
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@s3 = AWS::S3.new({access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key]})
|
6
|
+
@bucket = get_bucket(options[:bucket])
|
7
|
+
@store_prefix = options[:store_prefix] || ''
|
8
|
+
end
|
9
|
+
|
10
|
+
|
11
|
+
public
|
12
|
+
|
13
|
+
def upload(store_path, file_name)
|
14
|
+
key = File.join(get_full_store_path(store_path), File.basename(file_name))
|
15
|
+
|
16
|
+
checksum = get_signature(file_name)
|
17
|
+
|
18
|
+
begin
|
19
|
+
|
20
|
+
file = File.open(file_name, 'rb')
|
21
|
+
|
22
|
+
obj = @bucket.objects[key]
|
23
|
+
|
24
|
+
obj.write(:content_length => file.size, metadata: {checksum: checksum}) do |buffer, bytes|
|
25
|
+
buffer.write(file.read(bytes))
|
26
|
+
end
|
27
|
+
|
28
|
+
file.close
|
29
|
+
|
30
|
+
rescue Exception => error
|
31
|
+
raise "Error upload file <#{file_name}> to s3 <#{key}>: #{error.message}"
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_uploaded_files(prefix = '', limit = 100)
|
37
|
+
result = []
|
38
|
+
|
39
|
+
prefix = get_full_store_path(prefix)
|
40
|
+
|
41
|
+
@bucket.objects.with_prefix(prefix).each(:limit => limit) do |object|
|
42
|
+
unless object.key.end_with?('/')
|
43
|
+
result << object
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
result
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def get_bucket(bucket_name)
|
54
|
+
bucket = @s3.buckets[bucket_name]
|
55
|
+
|
56
|
+
unless bucket.exists?
|
57
|
+
raise "Bucket #{bucket_name} doesn't not exists. Please create it"
|
58
|
+
end
|
59
|
+
|
60
|
+
bucket
|
61
|
+
end
|
62
|
+
|
63
|
+
def get_signature(file_name)
|
64
|
+
Digest::MD5.hexdigest(File.size(file_name).to_s)
|
65
|
+
end
|
66
|
+
|
67
|
+
def get_full_store_path(path)
|
68
|
+
if @store_prefix.empty?
|
69
|
+
path
|
70
|
+
else
|
71
|
+
File.join(@store_prefix, path)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
export:
|
2
|
+
# List dbs for export
|
3
|
+
dbs: db1, db2
|
4
|
+
|
5
|
+
# List collections for export to csv.
|
6
|
+
# Format: array of name, [query], [fields].
|
7
|
+
# if collections empty than all collections with all fields will be export
|
8
|
+
collections:
|
9
|
+
- name: coll1
|
10
|
+
|
11
|
+
- name: coll2
|
12
|
+
query: '{field1: { $gte: 3 }, field2: 'value1'}'
|
13
|
+
fields: field1, field2, field3
|
14
|
+
|
15
|
+
|
16
|
+
# Temporary directory for files. Default: system temp directory
|
17
|
+
temp_directory:
|
18
|
+
|
19
|
+
# [minute] [hour] [day] [month] [weekday]
|
20
|
+
cron_time: 30 0 * * *
|
21
|
+
|
22
|
+
mongo:
|
23
|
+
host: 'localhost'
|
24
|
+
port: 27017
|
25
|
+
username:
|
26
|
+
password:
|
27
|
+
|
28
|
+
# If you do not specify an authentication database than database specified to export holds the user’s credentials
|
29
|
+
authentication_database: admin
|
30
|
+
|
31
|
+
|
32
|
+
s3:
|
33
|
+
access_key_id: 'access_key_id'
|
34
|
+
secret_access_key: 'secret_access_key'
|
35
|
+
bucket: 'export_mongo_s3-backups'
|
36
|
+
store_prefix:
|
data/lib/helpers/hash.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
class Hash
|
2
|
+
|
3
|
+
def symbolize_keys!
|
4
|
+
transform_keys! { |key| key.to_sym rescue key }
|
5
|
+
end
|
6
|
+
|
7
|
+
def deep_symbolize_keys!
|
8
|
+
deep_transform_keys!{ |key| key.to_sym rescue key }
|
9
|
+
end
|
10
|
+
|
11
|
+
def transform_keys!
|
12
|
+
keys.each do |key|
|
13
|
+
self[yield(key)] = delete(key)
|
14
|
+
end
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def deep_transform_keys!(&block)
|
19
|
+
keys.each do |key|
|
20
|
+
value = delete(key)
|
21
|
+
self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
|
22
|
+
end
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
data/lib/helpers/time.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
class Time
|
2
|
+
|
3
|
+
def midnight
|
4
|
+
change(:hour => 0)
|
5
|
+
end
|
6
|
+
|
7
|
+
def change(options)
|
8
|
+
new_year = options.fetch(:year, year)
|
9
|
+
new_month = options.fetch(:month, month)
|
10
|
+
new_day = options.fetch(:day, day)
|
11
|
+
new_hour = options.fetch(:hour, hour)
|
12
|
+
new_min = options.fetch(:min, options[:hour] ? 0 : min)
|
13
|
+
new_sec = options.fetch(:sec, (options[:hour] || options[:min]) ? 0 : sec)
|
14
|
+
new_usec = options.fetch(:usec, (options[:hour] || options[:min] || options[:sec]) ? 0 : Rational(nsec, 1000))
|
15
|
+
|
16
|
+
if utc?
|
17
|
+
::Time.utc(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
18
|
+
elsif zone
|
19
|
+
::Time.local(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
20
|
+
else
|
21
|
+
::Time.new(new_year, new_month, new_day, new_hour, new_min, new_sec + (new_usec.to_r / 1000000), utc_offset)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: export_mongo_s3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yakupov Dima
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.6'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.6'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-sdk
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.57'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.57'
|
41
|
+
description: Command-line application for MongoDB export(mongoexport) to CSV and upload
|
42
|
+
to Amazon S3
|
43
|
+
email:
|
44
|
+
- yakupov.dima@mail.ru
|
45
|
+
executables:
|
46
|
+
- export_mongo_s3
|
47
|
+
extensions: []
|
48
|
+
extra_rdoc_files: []
|
49
|
+
files:
|
50
|
+
- .gitignore
|
51
|
+
- Gemfile
|
52
|
+
- README.md
|
53
|
+
- bin/export_mongo_s3
|
54
|
+
- export_mongo_s3.gemspec
|
55
|
+
- lib/export_mongo_s3.rb
|
56
|
+
- lib/export_mongo_s3/application.rb
|
57
|
+
- lib/export_mongo_s3/db.rb
|
58
|
+
- lib/export_mongo_s3/scheduler.rb
|
59
|
+
- lib/export_mongo_s3/storage.rb
|
60
|
+
- lib/helpers/config.yml.template
|
61
|
+
- lib/helpers/fixnum.rb
|
62
|
+
- lib/helpers/hash.rb
|
63
|
+
- lib/helpers/time.rb
|
64
|
+
homepage: ''
|
65
|
+
licenses:
|
66
|
+
- MIT
|
67
|
+
metadata: {}
|
68
|
+
post_install_message:
|
69
|
+
rdoc_options: []
|
70
|
+
require_paths:
|
71
|
+
- lib
|
72
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - '>='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
requirements: []
|
83
|
+
rubyforge_project:
|
84
|
+
rubygems_version: 2.4.2
|
85
|
+
signing_key:
|
86
|
+
specification_version: 4
|
87
|
+
summary: Some summary
|
88
|
+
test_files: []
|