export_mongo_s3 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +24 -0
- data/Gemfile +3 -0
- data/README.md +20 -0
- data/bin/export_mongo_s3 +11 -0
- data/export_mongo_s3.gemspec +19 -0
- data/lib/export_mongo_s3.rb +28 -0
- data/lib/export_mongo_s3/application.rb +226 -0
- data/lib/export_mongo_s3/db.rb +142 -0
- data/lib/export_mongo_s3/scheduler.rb +97 -0
- data/lib/export_mongo_s3/storage.rb +76 -0
- data/lib/helpers/config.yml.template +36 -0
- data/lib/helpers/fixnum.rb +11 -0
- data/lib/helpers/hash.rb +26 -0
- data/lib/helpers/time.rb +25 -0
- metadata +88 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d3c8c2fbe8679bb4b3c17ea0ceed2c86492a00c6
|
4
|
+
data.tar.gz: 7e2192e16878e2cd87a4f6e4fa247ca081e60284
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: d9f0bf7dcbd0500bddfbc24c7e8ed89e0f3ba034a582a715da5145a999c61410a98ff58edc03adb18c43e6ad45a72918425097c898a362603d9164281138b774
|
7
|
+
data.tar.gz: 2fba509d5f4ba909c62437e95914586a6fa812f8ab92a497d71dbd4ef4d8a45da1df562b8e43c48e6f9f28b6b58a734f5ec7a5fa470cc5818a0297e4ddf37f83
|
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
*.idea
|
4
|
+
.bundle
|
5
|
+
.config
|
6
|
+
.yardoc
|
7
|
+
Gemfile.lock
|
8
|
+
InstalledFiles
|
9
|
+
_yardoc
|
10
|
+
coverage
|
11
|
+
doc/
|
12
|
+
lib/bundler/man
|
13
|
+
pkg
|
14
|
+
rdoc
|
15
|
+
spec/reports
|
16
|
+
test/tmp
|
17
|
+
test/version_tmp
|
18
|
+
tmp
|
19
|
+
*.bundle
|
20
|
+
*.so
|
21
|
+
*.o
|
22
|
+
*.a
|
23
|
+
mkmf.log
|
24
|
+
*.yml
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# ExportMongoS3
|
2
|
+
|
3
|
+
Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
gem 'export_mongo_s3'
|
10
|
+
|
11
|
+
And then execute:
|
12
|
+
|
13
|
+
$ bundle
|
14
|
+
|
15
|
+
Or install it yourself as:
|
16
|
+
|
17
|
+
$ gem install export_mongo_s3
|
18
|
+
|
19
|
+
## Help
|
20
|
+
|
data/bin/export_mongo_s3
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Gem::Specification.new do |spec|
|
2
|
+
spec.name = 'export_mongo_s3'
|
3
|
+
spec.version = '0.0.1'
|
4
|
+
spec.authors = ['Yakupov Dima']
|
5
|
+
spec.email = ['yakupov.dima@mail.ru']
|
6
|
+
spec.summary = "Some summary"
|
7
|
+
spec.description = "Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3"
|
8
|
+
spec.homepage = ''
|
9
|
+
spec.license = 'MIT'
|
10
|
+
|
11
|
+
spec.files = `git ls-files -z`.split("\x0")
|
12
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
13
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
14
|
+
spec.require_paths = ["lib"]
|
15
|
+
|
16
|
+
spec.add_development_dependency 'bundler', '~> 1.6'
|
17
|
+
|
18
|
+
spec.add_runtime_dependency 'aws-sdk', '~> 1.57'
|
19
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'aws-sdk'
|
3
|
+
require 'fileutils'
|
4
|
+
require 'digest/md5'
|
5
|
+
require 'tmpdir'
|
6
|
+
|
7
|
+
require_relative 'export_mongo_s3/application'
|
8
|
+
require_relative 'export_mongo_s3/db'
|
9
|
+
require_relative 'export_mongo_s3/storage'
|
10
|
+
require_relative 'export_mongo_s3/scheduler'
|
11
|
+
|
12
|
+
require_relative 'helpers/fixnum'
|
13
|
+
require_relative 'helpers/hash'
|
14
|
+
require_relative 'helpers/time'
|
15
|
+
|
16
|
+
module ExportMongoS3
|
17
|
+
|
18
|
+
def self.name
|
19
|
+
File.basename( __FILE__, '.rb')
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.root_path
|
23
|
+
@root_path if @root_path
|
24
|
+
spec = Gem::Specification.find_by_name(self.name)
|
25
|
+
@root_path = spec.gem_dir
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,226 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Application
|
3
|
+
|
4
|
+
def initialize(argv)
|
5
|
+
@parser = OptionParser.new
|
6
|
+
|
7
|
+
@params = parse_options(argv)
|
8
|
+
@config = parse_config(@params[:config])
|
9
|
+
|
10
|
+
db_options = @config[:mongo]
|
11
|
+
db_options.merge!(collections: @config[:export][:collections])
|
12
|
+
|
13
|
+
@db = Db.new(db_options)
|
14
|
+
|
15
|
+
@storage = Storage.new(@config[:s3])
|
16
|
+
end
|
17
|
+
|
18
|
+
|
19
|
+
public
|
20
|
+
|
21
|
+
def run
|
22
|
+
|
23
|
+
case
|
24
|
+
when @params[:export_list] # EXPORT_LIST
|
25
|
+
show_uploaded_files(@params[:export_list])
|
26
|
+
|
27
|
+
when @params[:export_all] # EXPORT_ALL
|
28
|
+
dbs_str = @config[:export][:dbs]
|
29
|
+
|
30
|
+
if dbs_str.nil? || dbs_str.empty?
|
31
|
+
raise 'config.yml::export.dbs is empty'
|
32
|
+
end
|
33
|
+
|
34
|
+
db_names = dbs_str.split(',').each { |db_name| db_name.strip! }
|
35
|
+
|
36
|
+
export(db_names)
|
37
|
+
|
38
|
+
when @params[:export] # EXPORT
|
39
|
+
export([@params[:export]])
|
40
|
+
|
41
|
+
when @params[:cron_update] || @params[:cron_clear] # CRON
|
42
|
+
cron_options =
|
43
|
+
{
|
44
|
+
update: @params[:cron_update],
|
45
|
+
clear: @params[:cron_clear],
|
46
|
+
config: @params[:config],
|
47
|
+
time: @config[:backup][:cron_time],
|
48
|
+
}
|
49
|
+
|
50
|
+
Scheduler.new(cron_options).execute
|
51
|
+
|
52
|
+
else
|
53
|
+
puts "\n#{@parser}\n"
|
54
|
+
exit(0)
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def show_uploaded_files(db_name)
|
63
|
+
|
64
|
+
files = @storage.get_uploaded_files(db_name)
|
65
|
+
|
66
|
+
if files.empty?
|
67
|
+
puts 'Files not found'
|
68
|
+
else
|
69
|
+
|
70
|
+
puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')
|
71
|
+
|
72
|
+
files.each do |file|
|
73
|
+
|
74
|
+
file_name = file.key
|
75
|
+
file_size = file.content_length / 1024 / 1024
|
76
|
+
file_last_modified = file.last_modified
|
77
|
+
|
78
|
+
puts sprintf('%-30s %-15s %-20s', file_name, file_size, file_last_modified)
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def export(db_names)
|
85
|
+
|
86
|
+
db_names.each do |db_name|
|
87
|
+
|
88
|
+
puts "export db #{db_name.upcase}:"
|
89
|
+
|
90
|
+
tmp_dir = get_temp_dir
|
91
|
+
|
92
|
+
begin
|
93
|
+
|
94
|
+
export_path = File.join(tmp_dir, db_name)
|
95
|
+
|
96
|
+
puts "\t make csv..."
|
97
|
+
@db.export_db(db_name, export_path)
|
98
|
+
|
99
|
+
if Dir["#{export_path}/*"].empty?
|
100
|
+
puts "\t [skip] db is empty"
|
101
|
+
|
102
|
+
else
|
103
|
+
|
104
|
+
puts "\t compress..."
|
105
|
+
zip_result = %x(zip -6 -r '#{export_path}.zip' '#{export_path}/' -j)
|
106
|
+
raise "Error zip. Msg: #{zip_result}" unless $?.exitstatus.zero?
|
107
|
+
|
108
|
+
FileUtils.rm_rf(export_path)
|
109
|
+
|
110
|
+
zip_file_path = "#{export_path}.zip"
|
111
|
+
|
112
|
+
if File.exists?(zip_file_path)
|
113
|
+
puts "\t upload to s3..."
|
114
|
+
@storage.upload(db_name, zip_file_path)
|
115
|
+
|
116
|
+
File.delete(zip_file_path)
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
|
121
|
+
ensure
|
122
|
+
FileUtils.remove_entry_secure(tmp_dir)
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
puts '[done] export'
|
128
|
+
end
|
129
|
+
|
130
|
+
def create_config(path)
|
131
|
+
|
132
|
+
path = '.' if path.nil? || path == ''
|
133
|
+
|
134
|
+
file = File.join(path, 'config.yml')
|
135
|
+
|
136
|
+
if File.exists?(file)
|
137
|
+
raise "create_config: '#{file}' already exists"
|
138
|
+
elsif File.exists?(file.downcase)
|
139
|
+
raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
|
140
|
+
elsif !File.exists?(File.dirname(file))
|
141
|
+
raise "create_config: directory '#{File.dirname(file)}' does not exist"
|
142
|
+
else
|
143
|
+
file_template = File.join(ExportMongoS3.root_path, 'lib/helpers/config.yml.template')
|
144
|
+
|
145
|
+
FileUtils.cp file_template, file
|
146
|
+
end
|
147
|
+
|
148
|
+
puts "[done] file #{file} was created"
|
149
|
+
end
|
150
|
+
|
151
|
+
def parse_options(argv)
|
152
|
+
params = {}
|
153
|
+
|
154
|
+
@parser.on('--export_all', 'Export databases specified in config.yml and upload to S3 bucket') do
|
155
|
+
params[:export_all] = true
|
156
|
+
end
|
157
|
+
@parser.on('--export DB_NAME', String, 'Export database and upload to S3 bucket') do |db_name|
|
158
|
+
params[:export] = db_name
|
159
|
+
end
|
160
|
+
@parser.on('-l', '--list_exported [DB_NAME]', String, 'Show list of exported dbs') do |db_name|
|
161
|
+
params[:export_list] = db_name || ''
|
162
|
+
end
|
163
|
+
@parser.on('--write_cron', 'Add/update export_all job') do
|
164
|
+
params[:cron_update] = true
|
165
|
+
end
|
166
|
+
@parser.on('--clear_cron', 'Clear export_all job') do
|
167
|
+
params[:cron_clear] = true
|
168
|
+
end
|
169
|
+
@parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default: ./config.yml') do |path|
|
170
|
+
params[:config] = path || ''
|
171
|
+
end
|
172
|
+
@parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
|
173
|
+
create_config(path)
|
174
|
+
exit(0)
|
175
|
+
end
|
176
|
+
@parser.on('-h', '--help', 'Show help') do
|
177
|
+
puts "\n#{@parser}\n"
|
178
|
+
exit(0)
|
179
|
+
end
|
180
|
+
|
181
|
+
@parser.parse!(argv)
|
182
|
+
|
183
|
+
if [params[:export_all], params[:export], params[:export_list], params[:cron_update], params[:cron_clear]].compact.length > 1
|
184
|
+
raise 'Can only export_all, export, list_exported, write_cron or clear_cron. Choose one.'
|
185
|
+
end
|
186
|
+
|
187
|
+
if params[:config].nil? || params[:config] == ''
|
188
|
+
params[:config] = './config.yml'
|
189
|
+
end
|
190
|
+
|
191
|
+
params[:config] = File.absolute_path(params[:config])
|
192
|
+
|
193
|
+
params
|
194
|
+
end
|
195
|
+
|
196
|
+
def parse_config(config_path)
|
197
|
+
|
198
|
+
begin
|
199
|
+
config = YAML.load(File.read(config_path))
|
200
|
+
rescue Errno::ENOENT
|
201
|
+
raise "Could not find config file '#{config_path}'"
|
202
|
+
rescue ArgumentError => error
|
203
|
+
raise "Could not parse config file '#{config_path}' - #{error}"
|
204
|
+
end
|
205
|
+
|
206
|
+
config.deep_symbolize_keys!
|
207
|
+
|
208
|
+
raise 'config.yml. Section <export> not found' if config[:export].nil?
|
209
|
+
raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
|
210
|
+
raise 'config.yml. Section <s3> not found' if config[:s3].nil?
|
211
|
+
|
212
|
+
config
|
213
|
+
end
|
214
|
+
|
215
|
+
def get_temp_dir
|
216
|
+
temp_dir = @config[:export][:temp_directory]
|
217
|
+
|
218
|
+
if temp_dir.nil? || temp_dir == ''
|
219
|
+
temp_dir = Dir.tmpdir
|
220
|
+
end
|
221
|
+
|
222
|
+
Dir.mktmpdir(nil, temp_dir)
|
223
|
+
end
|
224
|
+
|
225
|
+
end
|
226
|
+
end
|
@@ -0,0 +1,142 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Db
|
3
|
+
|
4
|
+
SYSTEM_COLLECTIONS = %w(admin_users fs.chunks fs.files system.indexes)
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@options = options
|
8
|
+
@connection_options = connection(options)
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
public
|
13
|
+
|
14
|
+
def export_db(db, out_path)
|
15
|
+
|
16
|
+
collection_settings_map = prepared_collection_settings
|
17
|
+
|
18
|
+
if collection_settings_map.empty?
|
19
|
+
collection_names = get_collections(db)
|
20
|
+
else
|
21
|
+
collection_names = collection_settings_map.keys
|
22
|
+
end
|
23
|
+
|
24
|
+
collection_names.each do |collection_name|
|
25
|
+
|
26
|
+
collection_settings = collection_settings_map[collection_name]
|
27
|
+
|
28
|
+
if collection_settings.nil?
|
29
|
+
export(db, collection_name, out_path)
|
30
|
+
else
|
31
|
+
fields = collection_settings[:fields]
|
32
|
+
query = collection_settings[:query]
|
33
|
+
|
34
|
+
export(db, collection_name, out_path, fields, query)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def connection(options)
|
43
|
+
host = (options[:host].nil? || options[:host] == '') ? 'localhost' : options[:host]
|
44
|
+
port = options[:port].nil? ? 27017 : options[:port]
|
45
|
+
username = options[:username]
|
46
|
+
password = options[:password]
|
47
|
+
authentication_database = options[:authentication_database]
|
48
|
+
|
49
|
+
auth_options = ''
|
50
|
+
|
51
|
+
unless username.nil? || username == '' || password.nil? || password == ''
|
52
|
+
auth_options = "-u '#{username}' -p '#{password}'"
|
53
|
+
auth_options << " --authenticationDatabase '#{authentication_database}'" unless authentication_database.nil? || authentication_database == ''
|
54
|
+
end
|
55
|
+
|
56
|
+
"--host '#{host}' --port '#{port}' #{auth_options}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def export(db, collection, out_path, fields = [], query = nil)
|
60
|
+
|
61
|
+
if fields.empty?
|
62
|
+
fields = get_fields(db, collection)
|
63
|
+
end
|
64
|
+
|
65
|
+
return if fields.empty?
|
66
|
+
|
67
|
+
command = 'mongoexport'
|
68
|
+
command << " #{@connection_options}"
|
69
|
+
command << ' --csv'
|
70
|
+
command << " --db '#{db}'"
|
71
|
+
command << " --collection '#{collection}'"
|
72
|
+
command << " --fields '#{fields.join(',')}'"
|
73
|
+
command << " --query '#{query}'" unless query.nil?
|
74
|
+
command << " --out '#{out_path}/#{collection}.csv'"
|
75
|
+
command << ' > /dev/null'
|
76
|
+
|
77
|
+
system(command)
|
78
|
+
raise "Error mongoexport '#{db}'" unless $?.exitstatus.zero?
|
79
|
+
end
|
80
|
+
|
81
|
+
def get_collections(db)
|
82
|
+
command = 'mongo'
|
83
|
+
command << " #{@connection_options}"
|
84
|
+
command << " #{db} --eval 'rs.slaveOk(); db.getCollectionNames();'"
|
85
|
+
command << ' --quiet'
|
86
|
+
|
87
|
+
result = %x(#{command})
|
88
|
+
raise "Error get collections for db '#{db}'. Msg: #{result}" unless $?.exitstatus.zero?
|
89
|
+
|
90
|
+
result.strip.split(',') - SYSTEM_COLLECTIONS
|
91
|
+
end
|
92
|
+
|
93
|
+
def get_fields(db, collection)
|
94
|
+
command = 'mongo'
|
95
|
+
command << " #{@connection_options}"
|
96
|
+
command << " #{db} --eval 'rs.slaveOk(); var fields = []; for(var field in db.#{collection}.find().sort({_id: -1}).limit(1)[0]) { fields.push(field); }; fields;'"
|
97
|
+
command << ' --quiet'
|
98
|
+
|
99
|
+
result = %x(#{command})
|
100
|
+
raise "Error get fields for db '#{db}' and collection '#{collection}'. Msg: #{result}" unless $?.exitstatus.zero?
|
101
|
+
|
102
|
+
result.strip.split(',')
|
103
|
+
end
|
104
|
+
|
105
|
+
def prepared_collection_settings
|
106
|
+
@prepared_collection_settings if @prepared_collection_settings
|
107
|
+
|
108
|
+
collection_settings = @options[:collections]
|
109
|
+
|
110
|
+
prepared_settings = {}
|
111
|
+
|
112
|
+
if collection_settings.is_a?(Array)
|
113
|
+
collection_settings.each do |collection|
|
114
|
+
|
115
|
+
name = collection['name']
|
116
|
+
query = collection['query'].nil? || collection['query'] == '' ? nil : collection['query'].to_s
|
117
|
+
|
118
|
+
fields = if collection['fields'].is_a?(String)
|
119
|
+
collection['fields'].split(',').each { |field| field.strip! }
|
120
|
+
else
|
121
|
+
[]
|
122
|
+
end
|
123
|
+
|
124
|
+
|
125
|
+
if name.nil? || name == ''
|
126
|
+
raise "Not valid param <name: #{collection}>"
|
127
|
+
end
|
128
|
+
|
129
|
+
prepared_settings[name] =
|
130
|
+
{
|
131
|
+
name: name,
|
132
|
+
fields: fields,
|
133
|
+
query: query
|
134
|
+
}
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
@prepared_collection_settings = prepared_settings
|
139
|
+
end
|
140
|
+
|
141
|
+
end
|
142
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Scheduler
|
3
|
+
|
4
|
+
def initialize(options = {})
|
5
|
+
@options = options
|
6
|
+
|
7
|
+
if [@options[:write], @options[:clear]].compact.length > 1
|
8
|
+
raise 'cron: Can only write or clear. Choose one.'
|
9
|
+
end
|
10
|
+
|
11
|
+
unless @options[:time] =~ /\A[*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-6]+\z/
|
12
|
+
raise 'config.yml: cron_time is not valid'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute
|
17
|
+
write_crontab(updated_crontab)
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def read_crontab
|
23
|
+
return @read_crontab if @read_crontab
|
24
|
+
|
25
|
+
command = 'crontab -l'
|
26
|
+
|
27
|
+
command_results = %x[#{command} 2> /dev/null]
|
28
|
+
|
29
|
+
@read_crontab = $?.exitstatus.zero? ? prepare(command_results) : ''
|
30
|
+
end
|
31
|
+
|
32
|
+
def prepare(contents)
|
33
|
+
# Some cron implementations require all non-comment lines to be newline-
|
34
|
+
# terminated. (issue #95) Strip all newlines and replace with the default
|
35
|
+
# platform record seperator ($/)
|
36
|
+
contents.gsub!(/\s+$/, $/)
|
37
|
+
end
|
38
|
+
|
39
|
+
def write_crontab(contents)
|
40
|
+
command = 'crontab -'
|
41
|
+
|
42
|
+
IO.popen(command, 'r+') do |crontab|
|
43
|
+
crontab.write(contents)
|
44
|
+
crontab.close_write
|
45
|
+
end
|
46
|
+
|
47
|
+
success = $?.exitstatus.zero?
|
48
|
+
|
49
|
+
if success
|
50
|
+
action = @options[:update] ? 'updated' : 'cleared'
|
51
|
+
puts "[done] crontab file #{action}"
|
52
|
+
exit(0)
|
53
|
+
else
|
54
|
+
raise "Couldn't write crontab"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def updated_crontab
|
59
|
+
# Check for unopened or unclosed identifier blocks
|
60
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && (read_crontab =~ Regexp.new("^#{comment_close}\s*$")).nil?
|
61
|
+
raise "Unclosed indentifier; Your crontab file contains '#{comment_open}', but no '#{comment_close}'"
|
62
|
+
elsif (read_crontab =~ Regexp.new("^#{comment_open}\s*$")).nil? && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
63
|
+
raise "Unopened indentifier; Your crontab file contains '#{comment_close}', but no '#{comment_open}'"
|
64
|
+
end
|
65
|
+
|
66
|
+
# If an existing identier block is found, replace it with the new cron entries
|
67
|
+
if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
|
68
|
+
# If the existing crontab file contains backslashes they get lost going through gsub.
|
69
|
+
# .gsub('\\', '\\\\\\') preserves them. Go figure.
|
70
|
+
read_crontab.gsub(Regexp.new("^#{comment_open}\s*$.+^#{comment_close}\s*$", Regexp::MULTILINE), crontab_task.chomp.gsub('\\', '\\\\\\'))
|
71
|
+
else # Otherwise, append the new cron entries after any existing ones
|
72
|
+
[read_crontab, crontab_task].join("\n\n")
|
73
|
+
end.gsub(/\n{3,}/, "\n\n") # More than two newlines becomes just two.
|
74
|
+
end
|
75
|
+
|
76
|
+
def crontab_task
|
77
|
+
return '' if @options[:clear]
|
78
|
+
[comment_open, crontab_job, comment_close].compact.join("\n") + "\n"
|
79
|
+
end
|
80
|
+
|
81
|
+
def crontab_job
|
82
|
+
"#{@options[:time]} /bin/bash -l -c '#{ExportMongoS3.name} --export_all --config #{@options[:config]}'"
|
83
|
+
end
|
84
|
+
|
85
|
+
def comment_base
|
86
|
+
"#{ExportMongoS3.name} task"
|
87
|
+
end
|
88
|
+
|
89
|
+
def comment_open
|
90
|
+
"# Begin #{comment_base}"
|
91
|
+
end
|
92
|
+
|
93
|
+
def comment_close
|
94
|
+
"# End #{comment_base}"
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module ExportMongoS3
|
2
|
+
class Storage
|
3
|
+
|
4
|
+
def initialize(options)
|
5
|
+
@s3 = AWS::S3.new({access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key]})
|
6
|
+
@bucket = get_bucket(options[:bucket])
|
7
|
+
@store_prefix = options[:store_prefix] || ''
|
8
|
+
end
|
9
|
+
|
10
|
+
|
11
|
+
public
|
12
|
+
|
13
|
+
def upload(store_path, file_name)
|
14
|
+
key = File.join(get_full_store_path(store_path), File.basename(file_name))
|
15
|
+
|
16
|
+
checksum = get_signature(file_name)
|
17
|
+
|
18
|
+
begin
|
19
|
+
|
20
|
+
file = File.open(file_name, 'rb')
|
21
|
+
|
22
|
+
obj = @bucket.objects[key]
|
23
|
+
|
24
|
+
obj.write(:content_length => file.size, metadata: {checksum: checksum}) do |buffer, bytes|
|
25
|
+
buffer.write(file.read(bytes))
|
26
|
+
end
|
27
|
+
|
28
|
+
file.close
|
29
|
+
|
30
|
+
rescue Exception => error
|
31
|
+
raise "Error upload file <#{file_name}> to s3 <#{key}>: #{error.message}"
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_uploaded_files(prefix = '', limit = 100)
|
37
|
+
result = []
|
38
|
+
|
39
|
+
prefix = get_full_store_path(prefix)
|
40
|
+
|
41
|
+
@bucket.objects.with_prefix(prefix).each(:limit => limit) do |object|
|
42
|
+
unless object.key.end_with?('/')
|
43
|
+
result << object
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
result
|
48
|
+
end
|
49
|
+
|
50
|
+
|
51
|
+
private
|
52
|
+
|
53
|
+
def get_bucket(bucket_name)
|
54
|
+
bucket = @s3.buckets[bucket_name]
|
55
|
+
|
56
|
+
unless bucket.exists?
|
57
|
+
raise "Bucket #{bucket_name} doesn't not exists. Please create it"
|
58
|
+
end
|
59
|
+
|
60
|
+
bucket
|
61
|
+
end
|
62
|
+
|
63
|
+
def get_signature(file_name)
|
64
|
+
Digest::MD5.hexdigest(File.size(file_name).to_s)
|
65
|
+
end
|
66
|
+
|
67
|
+
def get_full_store_path(path)
|
68
|
+
if @store_prefix.empty?
|
69
|
+
path
|
70
|
+
else
|
71
|
+
File.join(@store_prefix, path)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
export:
|
2
|
+
# List dbs for export
|
3
|
+
dbs: db1, db2
|
4
|
+
|
5
|
+
# List collections for export to csv.
|
6
|
+
# Format: array of name, [query], [fields].
|
7
|
+
# if collections empty than all collections with all fields will be export
|
8
|
+
collections:
|
9
|
+
- name: coll1
|
10
|
+
|
11
|
+
- name: coll2
|
12
|
+
query: '{field1: { $gte: 3 }, field2: 'value1'}'
|
13
|
+
fields: field1, field2, field3
|
14
|
+
|
15
|
+
|
16
|
+
# Temporary directory for files. Default: system temp directory
|
17
|
+
temp_directory:
|
18
|
+
|
19
|
+
# [minute] [hour] [day] [month] [weekday]
|
20
|
+
cron_time: 30 0 * * *
|
21
|
+
|
22
|
+
mongo:
|
23
|
+
host: 'localhost'
|
24
|
+
port: 27017
|
25
|
+
username:
|
26
|
+
password:
|
27
|
+
|
28
|
+
# If you do not specify an authentication database than database specified to export holds the user’s credentials
|
29
|
+
authentication_database: admin
|
30
|
+
|
31
|
+
|
32
|
+
s3:
|
33
|
+
access_key_id: 'access_key_id'
|
34
|
+
secret_access_key: 'secret_access_key'
|
35
|
+
bucket: 'export_mongo_s3-backups'
|
36
|
+
store_prefix:
|
data/lib/helpers/hash.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
class Hash
|
2
|
+
|
3
|
+
def symbolize_keys!
|
4
|
+
transform_keys! { |key| key.to_sym rescue key }
|
5
|
+
end
|
6
|
+
|
7
|
+
def deep_symbolize_keys!
|
8
|
+
deep_transform_keys!{ |key| key.to_sym rescue key }
|
9
|
+
end
|
10
|
+
|
11
|
+
def transform_keys!
|
12
|
+
keys.each do |key|
|
13
|
+
self[yield(key)] = delete(key)
|
14
|
+
end
|
15
|
+
self
|
16
|
+
end
|
17
|
+
|
18
|
+
def deep_transform_keys!(&block)
|
19
|
+
keys.each do |key|
|
20
|
+
value = delete(key)
|
21
|
+
self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
|
22
|
+
end
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
data/lib/helpers/time.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
class Time
|
2
|
+
|
3
|
+
def midnight
|
4
|
+
change(:hour => 0)
|
5
|
+
end
|
6
|
+
|
7
|
+
def change(options)
|
8
|
+
new_year = options.fetch(:year, year)
|
9
|
+
new_month = options.fetch(:month, month)
|
10
|
+
new_day = options.fetch(:day, day)
|
11
|
+
new_hour = options.fetch(:hour, hour)
|
12
|
+
new_min = options.fetch(:min, options[:hour] ? 0 : min)
|
13
|
+
new_sec = options.fetch(:sec, (options[:hour] || options[:min]) ? 0 : sec)
|
14
|
+
new_usec = options.fetch(:usec, (options[:hour] || options[:min] || options[:sec]) ? 0 : Rational(nsec, 1000))
|
15
|
+
|
16
|
+
if utc?
|
17
|
+
::Time.utc(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
18
|
+
elsif zone
|
19
|
+
::Time.local(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
|
20
|
+
else
|
21
|
+
::Time.new(new_year, new_month, new_day, new_hour, new_min, new_sec + (new_usec.to_r / 1000000), utc_offset)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
metadata
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: export_mongo_s3
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Yakupov Dima
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.6'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.6'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-sdk
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.57'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.57'
|
41
|
+
description: Command-line application for MongoDB export(mongoexport) to CSV and upload
|
42
|
+
to Amazon S3
|
43
|
+
email:
|
44
|
+
- yakupov.dima@mail.ru
|
45
|
+
executables:
|
46
|
+
- export_mongo_s3
|
47
|
+
extensions: []
|
48
|
+
extra_rdoc_files: []
|
49
|
+
files:
|
50
|
+
- .gitignore
|
51
|
+
- Gemfile
|
52
|
+
- README.md
|
53
|
+
- bin/export_mongo_s3
|
54
|
+
- export_mongo_s3.gemspec
|
55
|
+
- lib/export_mongo_s3.rb
|
56
|
+
- lib/export_mongo_s3/application.rb
|
57
|
+
- lib/export_mongo_s3/db.rb
|
58
|
+
- lib/export_mongo_s3/scheduler.rb
|
59
|
+
- lib/export_mongo_s3/storage.rb
|
60
|
+
- lib/helpers/config.yml.template
|
61
|
+
- lib/helpers/fixnum.rb
|
62
|
+
- lib/helpers/hash.rb
|
63
|
+
- lib/helpers/time.rb
|
64
|
+
homepage: ''
|
65
|
+
licenses:
|
66
|
+
- MIT
|
67
|
+
metadata: {}
|
68
|
+
post_install_message:
|
69
|
+
rdoc_options: []
|
70
|
+
require_paths:
|
71
|
+
- lib
|
72
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - '>='
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '0'
|
82
|
+
requirements: []
|
83
|
+
rubyforge_project:
|
84
|
+
rubygems_version: 2.4.2
|
85
|
+
signing_key:
|
86
|
+
specification_version: 4
|
87
|
+
summary: Some summary
|
88
|
+
test_files: []
|