export_mongo_s3 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d3c8c2fbe8679bb4b3c17ea0ceed2c86492a00c6
4
+ data.tar.gz: 7e2192e16878e2cd87a4f6e4fa247ca081e60284
5
+ SHA512:
6
+ metadata.gz: d9f0bf7dcbd0500bddfbc24c7e8ed89e0f3ba034a582a715da5145a999c61410a98ff58edc03adb18c43e6ad45a72918425097c898a362603d9164281138b774
7
+ data.tar.gz: 2fba509d5f4ba909c62437e95914586a6fa812f8ab92a497d71dbd4ef4d8a45da1df562b8e43c48e6f9f28b6b58a734f5ec7a5fa470cc5818a0297e4ddf37f83
@@ -0,0 +1,24 @@
1
+ *.gem
2
+ *.rbc
3
+ *.idea
4
+ .bundle
5
+ .config
6
+ .yardoc
7
+ Gemfile.lock
8
+ InstalledFiles
9
+ _yardoc
10
+ coverage
11
+ doc/
12
+ lib/bundler/man
13
+ pkg
14
+ rdoc
15
+ spec/reports
16
+ test/tmp
17
+ test/version_tmp
18
+ tmp
19
+ *.bundle
20
+ *.so
21
+ *.o
22
+ *.a
23
+ mkmf.log
24
+ *.yml
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'aws-sdk'
@@ -0,0 +1,20 @@
1
+ # ExportMongoS3
2
+
3
+ Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'export_mongo_s3'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install export_mongo_s3
18
+
19
+ ## Help
20
+
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/export_mongo_s3'
4
+
5
+ begin
6
+ ExportMongoS3::Application.new(ARGV).run
7
+ rescue SystemExit
8
+ # do nothing
9
+ rescue Exception => error
10
+ abort "[abort] #{error.message}"
11
+ end
@@ -0,0 +1,19 @@
1
+ Gem::Specification.new do |spec|
2
+ spec.name = 'export_mongo_s3'
3
+ spec.version = '0.0.1'
4
+ spec.authors = ['Yakupov Dima']
5
+ spec.email = ['yakupov.dima@mail.ru']
6
+ spec.summary = "Some summary"
7
+ spec.description = "Command-line application for MongoDB export(mongoexport) to CSV and upload to Amazon S3"
8
+ spec.homepage = ''
9
+ spec.license = 'MIT'
10
+
11
+ spec.files = `git ls-files -z`.split("\x0")
12
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
13
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
14
+ spec.require_paths = ["lib"]
15
+
16
+ spec.add_development_dependency 'bundler', '~> 1.6'
17
+
18
+ spec.add_runtime_dependency 'aws-sdk', '~> 1.57'
19
+ end
@@ -0,0 +1,28 @@
1
+ require 'optparse'
2
+ require 'aws-sdk'
3
+ require 'fileutils'
4
+ require 'digest/md5'
5
+ require 'tmpdir'
6
+
7
+ require_relative 'export_mongo_s3/application'
8
+ require_relative 'export_mongo_s3/db'
9
+ require_relative 'export_mongo_s3/storage'
10
+ require_relative 'export_mongo_s3/scheduler'
11
+
12
+ require_relative 'helpers/fixnum'
13
+ require_relative 'helpers/hash'
14
+ require_relative 'helpers/time'
15
+
16
+ module ExportMongoS3
17
+
18
+ def self.name
19
+ File.basename( __FILE__, '.rb')
20
+ end
21
+
22
+ def self.root_path
23
+ @root_path if @root_path
24
+ spec = Gem::Specification.find_by_name(self.name)
25
+ @root_path = spec.gem_dir
26
+ end
27
+ end
28
+
@@ -0,0 +1,226 @@
1
+ module ExportMongoS3
2
+ class Application
3
+
4
+ def initialize(argv)
5
+ @parser = OptionParser.new
6
+
7
+ @params = parse_options(argv)
8
+ @config = parse_config(@params[:config])
9
+
10
+ db_options = @config[:mongo]
11
+ db_options.merge!(collections: @config[:export][:collections])
12
+
13
+ @db = Db.new(db_options)
14
+
15
+ @storage = Storage.new(@config[:s3])
16
+ end
17
+
18
+
19
+ public
20
+
21
+ def run
22
+
23
+ case
24
+ when @params[:export_list] # EXPORT_LIST
25
+ show_uploaded_files(@params[:export_list])
26
+
27
+ when @params[:export_all] # EXPORT_ALL
28
+ dbs_str = @config[:export][:dbs]
29
+
30
+ if dbs_str.nil? || dbs_str.empty?
31
+ raise 'config.yml::export.dbs is empty'
32
+ end
33
+
34
+ db_names = dbs_str.split(',').each { |db_name| db_name.strip! }
35
+
36
+ export(db_names)
37
+
38
+ when @params[:export] # EXPORT
39
+ export([@params[:export]])
40
+
41
+ when @params[:cron_update] || @params[:cron_clear] # CRON
42
+ cron_options =
43
+ {
44
+ update: @params[:cron_update],
45
+ clear: @params[:cron_clear],
46
+ config: @params[:config],
47
+ time: @config[:backup][:cron_time],
48
+ }
49
+
50
+ Scheduler.new(cron_options).execute
51
+
52
+ else
53
+ puts "\n#{@parser}\n"
54
+ exit(0)
55
+ end
56
+
57
+ end
58
+
59
+
60
+ private
61
+
62
+ def show_uploaded_files(db_name)
63
+
64
+ files = @storage.get_uploaded_files(db_name)
65
+
66
+ if files.empty?
67
+ puts 'Files not found'
68
+ else
69
+
70
+ puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')
71
+
72
+ files.each do |file|
73
+
74
+ file_name = file.key
75
+ file_size = file.content_length / 1024 / 1024
76
+ file_last_modified = file.last_modified
77
+
78
+ puts sprintf('%-30s %-15s %-20s', file_name, file_size, file_last_modified)
79
+ end
80
+
81
+ end
82
+ end
83
+
84
+ def export(db_names)
85
+
86
+ db_names.each do |db_name|
87
+
88
+ puts "export db #{db_name.upcase}:"
89
+
90
+ tmp_dir = get_temp_dir
91
+
92
+ begin
93
+
94
+ export_path = File.join(tmp_dir, db_name)
95
+
96
+ puts "\t make csv..."
97
+ @db.export_db(db_name, export_path)
98
+
99
+ if Dir["#{export_path}/*"].empty?
100
+ puts "\t [skip] db is empty"
101
+
102
+ else
103
+
104
+ puts "\t compress..."
105
+ zip_result = %x(zip -6 -r '#{export_path}.zip' '#{export_path}/' -j)
106
+ raise "Error zip. Msg: #{zip_result}" unless $?.exitstatus.zero?
107
+
108
+ FileUtils.rm_rf(export_path)
109
+
110
+ zip_file_path = "#{export_path}.zip"
111
+
112
+ if File.exists?(zip_file_path)
113
+ puts "\t upload to s3..."
114
+ @storage.upload(db_name, zip_file_path)
115
+
116
+ File.delete(zip_file_path)
117
+ end
118
+
119
+ end
120
+
121
+ ensure
122
+ FileUtils.remove_entry_secure(tmp_dir)
123
+ end
124
+
125
+ end
126
+
127
+ puts '[done] export'
128
+ end
129
+
130
+ def create_config(path)
131
+
132
+ path = '.' if path.nil? || path == ''
133
+
134
+ file = File.join(path, 'config.yml')
135
+
136
+ if File.exists?(file)
137
+ raise "create_config: '#{file}' already exists"
138
+ elsif File.exists?(file.downcase)
139
+ raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
140
+ elsif !File.exists?(File.dirname(file))
141
+ raise "create_config: directory '#{File.dirname(file)}' does not exist"
142
+ else
143
+ file_template = File.join(ExportMongoS3.root_path, 'lib/helpers/config.yml.template')
144
+
145
+ FileUtils.cp file_template, file
146
+ end
147
+
148
+ puts "[done] file #{file} was created"
149
+ end
150
+
151
+ def parse_options(argv)
152
+ params = {}
153
+
154
+ @parser.on('--export_all', 'Export databases specified in config.yml and upload to S3 bucket') do
155
+ params[:export_all] = true
156
+ end
157
+ @parser.on('--export DB_NAME', String, 'Export database and upload to S3 bucket') do |db_name|
158
+ params[:export] = db_name
159
+ end
160
+ @parser.on('-l', '--list_exported [DB_NAME]', String, 'Show list of exported dbs') do |db_name|
161
+ params[:export_list] = db_name || ''
162
+ end
163
+ @parser.on('--write_cron', 'Add/update export_all job') do
164
+ params[:cron_update] = true
165
+ end
166
+ @parser.on('--clear_cron', 'Clear export_all job') do
167
+ params[:cron_clear] = true
168
+ end
169
+ @parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default: ./config.yml') do |path|
170
+ params[:config] = path || ''
171
+ end
172
+ @parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
173
+ create_config(path)
174
+ exit(0)
175
+ end
176
+ @parser.on('-h', '--help', 'Show help') do
177
+ puts "\n#{@parser}\n"
178
+ exit(0)
179
+ end
180
+
181
+ @parser.parse!(argv)
182
+
183
+ if [params[:export_all], params[:export], params[:export_list], params[:cron_update], params[:cron_clear]].compact.length > 1
184
+ raise 'Can only export_all, export, list_exported, write_cron or clear_cron. Choose one.'
185
+ end
186
+
187
+ if params[:config].nil? || params[:config] == ''
188
+ params[:config] = './config.yml'
189
+ end
190
+
191
+ params[:config] = File.absolute_path(params[:config])
192
+
193
+ params
194
+ end
195
+
196
+ def parse_config(config_path)
197
+
198
+ begin
199
+ config = YAML.load(File.read(config_path))
200
+ rescue Errno::ENOENT
201
+ raise "Could not find config file '#{config_path}'"
202
+ rescue ArgumentError => error
203
+ raise "Could not parse config file '#{config_path}' - #{error}"
204
+ end
205
+
206
+ config.deep_symbolize_keys!
207
+
208
+ raise 'config.yml. Section <export> not found' if config[:export].nil?
209
+ raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
210
+ raise 'config.yml. Section <s3> not found' if config[:s3].nil?
211
+
212
+ config
213
+ end
214
+
215
+ def get_temp_dir
216
+ temp_dir = @config[:export][:temp_directory]
217
+
218
+ if temp_dir.nil? || temp_dir == ''
219
+ temp_dir = Dir.tmpdir
220
+ end
221
+
222
+ Dir.mktmpdir(nil, temp_dir)
223
+ end
224
+
225
+ end
226
+ end
@@ -0,0 +1,142 @@
1
+ module ExportMongoS3
2
+ class Db
3
+
4
+ SYSTEM_COLLECTIONS = %w(admin_users fs.chunks fs.files system.indexes)
5
+
6
+ def initialize(options)
7
+ @options = options
8
+ @connection_options = connection(options)
9
+ end
10
+
11
+
12
+ public
13
+
14
+ def export_db(db, out_path)
15
+
16
+ collection_settings_map = prepared_collection_settings
17
+
18
+ if collection_settings_map.empty?
19
+ collection_names = get_collections(db)
20
+ else
21
+ collection_names = collection_settings_map.keys
22
+ end
23
+
24
+ collection_names.each do |collection_name|
25
+
26
+ collection_settings = collection_settings_map[collection_name]
27
+
28
+ if collection_settings.nil?
29
+ export(db, collection_name, out_path)
30
+ else
31
+ fields = collection_settings[:fields]
32
+ query = collection_settings[:query]
33
+
34
+ export(db, collection_name, out_path, fields, query)
35
+ end
36
+ end
37
+
38
+ end
39
+
40
+ private
41
+
42
+ def connection(options)
43
+ host = (options[:host].nil? || options[:host] == '') ? 'localhost' : options[:host]
44
+ port = options[:port].nil? ? 27017 : options[:port]
45
+ username = options[:username]
46
+ password = options[:password]
47
+ authentication_database = options[:authentication_database]
48
+
49
+ auth_options = ''
50
+
51
+ unless username.nil? || username == '' || password.nil? || password == ''
52
+ auth_options = "-u '#{username}' -p '#{password}'"
53
+ auth_options << " --authenticationDatabase '#{authentication_database}'" unless authentication_database.nil? || authentication_database == ''
54
+ end
55
+
56
+ "--host '#{host}' --port '#{port}' #{auth_options}"
57
+ end
58
+
59
+ def export(db, collection, out_path, fields = [], query = nil)
60
+
61
+ if fields.empty?
62
+ fields = get_fields(db, collection)
63
+ end
64
+
65
+ return if fields.empty?
66
+
67
+ command = 'mongoexport'
68
+ command << " #{@connection_options}"
69
+ command << ' --csv'
70
+ command << " --db '#{db}'"
71
+ command << " --collection '#{collection}'"
72
+ command << " --fields '#{fields.join(',')}'"
73
+ command << " --query '#{query}'" unless query.nil?
74
+ command << " --out '#{out_path}/#{collection}.csv'"
75
+ command << ' > /dev/null'
76
+
77
+ system(command)
78
+ raise "Error mongoexport '#{db}'" unless $?.exitstatus.zero?
79
+ end
80
+
81
+ def get_collections(db)
82
+ command = 'mongo'
83
+ command << " #{@connection_options}"
84
+ command << " #{db} --eval 'rs.slaveOk(); db.getCollectionNames();'"
85
+ command << ' --quiet'
86
+
87
+ result = %x(#{command})
88
+ raise "Error get collections for db '#{db}'. Msg: #{result}" unless $?.exitstatus.zero?
89
+
90
+ result.strip.split(',') - SYSTEM_COLLECTIONS
91
+ end
92
+
93
+ def get_fields(db, collection)
94
+ command = 'mongo'
95
+ command << " #{@connection_options}"
96
+ command << " #{db} --eval 'rs.slaveOk(); var fields = []; for(var field in db.#{collection}.find().sort({_id: -1}).limit(1)[0]) { fields.push(field); }; fields;'"
97
+ command << ' --quiet'
98
+
99
+ result = %x(#{command})
100
+ raise "Error get fields for db '#{db}' and collection '#{collection}'. Msg: #{result}" unless $?.exitstatus.zero?
101
+
102
+ result.strip.split(',')
103
+ end
104
+
105
+ def prepared_collection_settings
106
+ @prepared_collection_settings if @prepared_collection_settings
107
+
108
+ collection_settings = @options[:collections]
109
+
110
+ prepared_settings = {}
111
+
112
+ if collection_settings.is_a?(Array)
113
+ collection_settings.each do |collection|
114
+
115
+ name = collection['name']
116
+ query = collection['query'].nil? || collection['query'] == '' ? nil : collection['query'].to_s
117
+
118
+ fields = if collection['fields'].is_a?(String)
119
+ collection['fields'].split(',').each { |field| field.strip! }
120
+ else
121
+ []
122
+ end
123
+
124
+
125
+ if name.nil? || name == ''
126
+ raise "Not valid param <name: #{collection}>"
127
+ end
128
+
129
+ prepared_settings[name] =
130
+ {
131
+ name: name,
132
+ fields: fields,
133
+ query: query
134
+ }
135
+ end
136
+ end
137
+
138
+ @prepared_collection_settings = prepared_settings
139
+ end
140
+
141
+ end
142
+ end
@@ -0,0 +1,97 @@
1
+ module ExportMongoS3
2
+ class Scheduler
3
+
4
+ def initialize(options = {})
5
+ @options = options
6
+
7
+ if [@options[:write], @options[:clear]].compact.length > 1
8
+ raise 'cron: Can only write or clear. Choose one.'
9
+ end
10
+
11
+ unless @options[:time] =~ /\A[*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-9]+ [*\-,0-6]+\z/
12
+ raise 'config.yml: cron_time is not valid'
13
+ end
14
+ end
15
+
16
+ def execute
17
+ write_crontab(updated_crontab)
18
+ end
19
+
20
+ private
21
+
22
+ def read_crontab
23
+ return @read_crontab if @read_crontab
24
+
25
+ command = 'crontab -l'
26
+
27
+ command_results = %x[#{command} 2> /dev/null]
28
+
29
+ @read_crontab = $?.exitstatus.zero? ? prepare(command_results) : ''
30
+ end
31
+
32
+ def prepare(contents)
33
+ # Some cron implementations require all non-comment lines to be newline-
34
+ # terminated. (issue #95) Strip all newlines and replace with the default
35
+ # platform record seperator ($/)
36
+ contents.gsub!(/\s+$/, $/)
37
+ end
38
+
39
+ def write_crontab(contents)
40
+ command = 'crontab -'
41
+
42
+ IO.popen(command, 'r+') do |crontab|
43
+ crontab.write(contents)
44
+ crontab.close_write
45
+ end
46
+
47
+ success = $?.exitstatus.zero?
48
+
49
+ if success
50
+ action = @options[:update] ? 'updated' : 'cleared'
51
+ puts "[done] crontab file #{action}"
52
+ exit(0)
53
+ else
54
+ raise "Couldn't write crontab"
55
+ end
56
+ end
57
+
58
+ def updated_crontab
59
+ # Check for unopened or unclosed identifier blocks
60
+ if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && (read_crontab =~ Regexp.new("^#{comment_close}\s*$")).nil?
61
+ raise "Unclosed indentifier; Your crontab file contains '#{comment_open}', but no '#{comment_close}'"
62
+ elsif (read_crontab =~ Regexp.new("^#{comment_open}\s*$")).nil? && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
63
+ raise "Unopened indentifier; Your crontab file contains '#{comment_close}', but no '#{comment_open}'"
64
+ end
65
+
66
+ # If an existing identier block is found, replace it with the new cron entries
67
+ if read_crontab =~ Regexp.new("^#{comment_open}\s*$") && read_crontab =~ Regexp.new("^#{comment_close}\s*$")
68
+ # If the existing crontab file contains backslashes they get lost going through gsub.
69
+ # .gsub('\\', '\\\\\\') preserves them. Go figure.
70
+ read_crontab.gsub(Regexp.new("^#{comment_open}\s*$.+^#{comment_close}\s*$", Regexp::MULTILINE), crontab_task.chomp.gsub('\\', '\\\\\\'))
71
+ else # Otherwise, append the new cron entries after any existing ones
72
+ [read_crontab, crontab_task].join("\n\n")
73
+ end.gsub(/\n{3,}/, "\n\n") # More than two newlines becomes just two.
74
+ end
75
+
76
+ def crontab_task
77
+ return '' if @options[:clear]
78
+ [comment_open, crontab_job, comment_close].compact.join("\n") + "\n"
79
+ end
80
+
81
+ def crontab_job
82
+ "#{@options[:time]} /bin/bash -l -c '#{ExportMongoS3.name} --export_all --config #{@options[:config]}'"
83
+ end
84
+
85
+ def comment_base
86
+ "#{ExportMongoS3.name} task"
87
+ end
88
+
89
+ def comment_open
90
+ "# Begin #{comment_base}"
91
+ end
92
+
93
+ def comment_close
94
+ "# End #{comment_base}"
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,76 @@
1
+ module ExportMongoS3
2
+ class Storage
3
+
4
+ def initialize(options)
5
+ @s3 = AWS::S3.new({access_key_id: options[:access_key_id], secret_access_key: options[:secret_access_key]})
6
+ @bucket = get_bucket(options[:bucket])
7
+ @store_prefix = options[:store_prefix] || ''
8
+ end
9
+
10
+
11
+ public
12
+
13
+ def upload(store_path, file_name)
14
+ key = File.join(get_full_store_path(store_path), File.basename(file_name))
15
+
16
+ checksum = get_signature(file_name)
17
+
18
+ begin
19
+
20
+ file = File.open(file_name, 'rb')
21
+
22
+ obj = @bucket.objects[key]
23
+
24
+ obj.write(:content_length => file.size, metadata: {checksum: checksum}) do |buffer, bytes|
25
+ buffer.write(file.read(bytes))
26
+ end
27
+
28
+ file.close
29
+
30
+ rescue Exception => error
31
+ raise "Error upload file <#{file_name}> to s3 <#{key}>: #{error.message}"
32
+ end
33
+
34
+ end
35
+
36
+ def get_uploaded_files(prefix = '', limit = 100)
37
+ result = []
38
+
39
+ prefix = get_full_store_path(prefix)
40
+
41
+ @bucket.objects.with_prefix(prefix).each(:limit => limit) do |object|
42
+ unless object.key.end_with?('/')
43
+ result << object
44
+ end
45
+ end
46
+
47
+ result
48
+ end
49
+
50
+
51
+ private
52
+
53
+ def get_bucket(bucket_name)
54
+ bucket = @s3.buckets[bucket_name]
55
+
56
+ unless bucket.exists?
57
+ raise "Bucket #{bucket_name} doesn't not exists. Please create it"
58
+ end
59
+
60
+ bucket
61
+ end
62
+
63
+ def get_signature(file_name)
64
+ Digest::MD5.hexdigest(File.size(file_name).to_s)
65
+ end
66
+
67
+ def get_full_store_path(path)
68
+ if @store_prefix.empty?
69
+ path
70
+ else
71
+ File.join(@store_prefix, path)
72
+ end
73
+ end
74
+
75
+ end
76
+ end
@@ -0,0 +1,36 @@
1
+ export:
2
+ # List dbs for export
3
+ dbs: db1, db2
4
+
5
+ # List collections for export to csv.
6
+ # Format: array of name, [query], [fields].
7
+ # if collections empty than all collections with all fields will be export
8
+ collections:
9
+ - name: coll1
10
+
11
+ - name: coll2
12
+ query: '{field1: { $gte: 3 }, field2: 'value1'}'
13
+ fields: field1, field2, field3
14
+
15
+
16
+ # Temporary directory for files. Default: system temp directory
17
+ temp_directory:
18
+
19
+ # [minute] [hour] [day] [month] [weekday]
20
+ cron_time: 30 0 * * *
21
+
22
+ mongo:
23
+ host: 'localhost'
24
+ port: 27017
25
+ username:
26
+ password:
27
+
28
+ # If you do not specify an authentication database than database specified to export holds the user’s credentials
29
+ authentication_database: admin
30
+
31
+
32
+ s3:
33
+ access_key_id: 'access_key_id'
34
+ secret_access_key: 'secret_access_key'
35
+ bucket: 'export_mongo_s3-backups'
36
+ store_prefix:
@@ -0,0 +1,11 @@
1
+ class Fixnum
2
+ SECONDS_IN_DAY = 24 * 60 * 60
3
+
4
+ def days
5
+ self * SECONDS_IN_DAY
6
+ end
7
+
8
+ def ago
9
+ Time.now - self
10
+ end
11
+ end
@@ -0,0 +1,26 @@
1
+ class Hash
2
+
3
+ def symbolize_keys!
4
+ transform_keys! { |key| key.to_sym rescue key }
5
+ end
6
+
7
+ def deep_symbolize_keys!
8
+ deep_transform_keys!{ |key| key.to_sym rescue key }
9
+ end
10
+
11
+ def transform_keys!
12
+ keys.each do |key|
13
+ self[yield(key)] = delete(key)
14
+ end
15
+ self
16
+ end
17
+
18
+ def deep_transform_keys!(&block)
19
+ keys.each do |key|
20
+ value = delete(key)
21
+ self[yield(key)] = value.is_a?(Hash) ? value.deep_transform_keys!(&block) : value
22
+ end
23
+ self
24
+ end
25
+
26
+ end
@@ -0,0 +1,25 @@
1
+ class Time
2
+
3
+ def midnight
4
+ change(:hour => 0)
5
+ end
6
+
7
+ def change(options)
8
+ new_year = options.fetch(:year, year)
9
+ new_month = options.fetch(:month, month)
10
+ new_day = options.fetch(:day, day)
11
+ new_hour = options.fetch(:hour, hour)
12
+ new_min = options.fetch(:min, options[:hour] ? 0 : min)
13
+ new_sec = options.fetch(:sec, (options[:hour] || options[:min]) ? 0 : sec)
14
+ new_usec = options.fetch(:usec, (options[:hour] || options[:min] || options[:sec]) ? 0 : Rational(nsec, 1000))
15
+
16
+ if utc?
17
+ ::Time.utc(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
18
+ elsif zone
19
+ ::Time.local(new_year, new_month, new_day, new_hour, new_min, new_sec, new_usec)
20
+ else
21
+ ::Time.new(new_year, new_month, new_day, new_hour, new_min, new_sec + (new_usec.to_r / 1000000), utc_offset)
22
+ end
23
+ end
24
+
25
+ end
metadata ADDED
@@ -0,0 +1,88 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: export_mongo_s3
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Yakupov Dima
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-12-02 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: aws-sdk
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '1.57'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '1.57'
41
+ description: Command-line application for MongoDB export(mongoexport) to CSV and upload
42
+ to Amazon S3
43
+ email:
44
+ - yakupov.dima@mail.ru
45
+ executables:
46
+ - export_mongo_s3
47
+ extensions: []
48
+ extra_rdoc_files: []
49
+ files:
50
+ - .gitignore
51
+ - Gemfile
52
+ - README.md
53
+ - bin/export_mongo_s3
54
+ - export_mongo_s3.gemspec
55
+ - lib/export_mongo_s3.rb
56
+ - lib/export_mongo_s3/application.rb
57
+ - lib/export_mongo_s3/db.rb
58
+ - lib/export_mongo_s3/scheduler.rb
59
+ - lib/export_mongo_s3/storage.rb
60
+ - lib/helpers/config.yml.template
61
+ - lib/helpers/fixnum.rb
62
+ - lib/helpers/hash.rb
63
+ - lib/helpers/time.rb
64
+ homepage: ''
65
+ licenses:
66
+ - MIT
67
+ metadata: {}
68
+ post_install_message:
69
+ rdoc_options: []
70
+ require_paths:
71
+ - lib
72
+ required_ruby_version: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - '>='
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ required_rubygems_version: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - '>='
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ requirements: []
83
+ rubyforge_project:
84
+ rubygems_version: 2.4.2
85
+ signing_key:
86
+ specification_version: 4
87
+ summary: Some summary
88
+ test_files: []