patriot-aws 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ YTA4NTk4NTg3NjFjZGQ2NTc3OTBkYWQyNzViMjcyNTkzYzFiZDZhMA==
5
+ data.tar.gz: !binary |-
6
+ ZGMwYmJhNDkwZGIwYzhlYjdjZjM2NTQwZjNlNDljNmUyMDBmMzVlYg==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MTA2NGJhMDEyNDNjNzhlOGQ1ZTY5MTI5Mzc5Y2FjZjBjNzljNzdmNmY2Zjc1
10
+ ODBlZmZjMDM1OWIxNDQxZjhlOGM0YzEyM2IzZWJiOGE1YjRmNTdhMzA2OGEx
11
+ NjQzZTdiMWI1YWYzM2EyODcwODg1ZTdlNWZkNGM4ZGMyNzJhNWE=
12
+ data.tar.gz: !binary |-
13
+ ODZmNTI4OWY4OWM1MzdjMDI0YzM0MWRhYjhmYWEwMmM4MGFhMjc5NTllMWRm
14
+ ODRhOTE2Zjg4NDNiYjY1NDY1NTg0ZTMwMWU2OTMyYmU1N2QxMTMzNjBkOGIz
15
+ OTkyOTU4ODgxMjBhNWNkMGM5OWY1MDhjZDAzZTM2OTkyZjRkMjE=
data/init.rb ADDED
@@ -0,0 +1,2 @@
1
+ require 'patriot'
2
+ require 'patriot_aws'
@@ -0,0 +1,248 @@
1
+ module PatriotAWS
2
+ module Command
3
+ class S3Command < Patriot::Command::Base
4
+ declare_command_name :s3
5
+ include PatriotAWS::Ext::AWS
6
+
7
+ command_attr :name, :name_suffix, :inifile,
8
+ :command, :src, :dest, :options
9
+
10
+ MODE_UPLOAD_FROM_LOCAL_TO_S3 = :mode_upload_from_local_to_s3
11
+ S3_PROTOCOLS = %w(s3 s3n s3a).freeze
12
+ COMMAND_COPY = :copy
13
+ S3_COMMANDS = [COMMAND_COPY].freeze
14
+
15
+ def job_id
16
+ job_id = "#{command_name}_#{@command}_#{@name}_#{@name_suffix}"
17
+ job_id
18
+ end
19
+
20
+ # @see Patriot::Command::Base#configure
21
+ def configure
22
+ @name_suffix ||= _date_
23
+ self
24
+ end
25
+
26
+ def execute
27
+ @logger.info "start s3 #{@command}"
28
+
29
+ @options ||= {}
30
+ @options = @options.symbolize_keys
31
+ @options = _set_options(@inifile, @options)
32
+
33
+ _check_attrs(@command, @options, @src, @dest)
34
+
35
+ config_aws(@options)
36
+ s3_cli = Aws::S3::Client.new
37
+
38
+ case @command
39
+ when COMMAND_COPY.to_s
40
+ _copy(s3_cli, @src, @dest, @options)
41
+ else
42
+ # should not reach here because this check is already done
43
+ # in configure
44
+ raise Exception,
45
+ 'command is invalid. '\
46
+ "supported commands are #{S3_COMMANDS.map(&:to_s)}"
47
+ end
48
+ end
49
+
50
+ # @private
51
+ # get inifile info and set options parameters
52
+ # @param String inifile
53
+ # @param Hash options
54
+ # @return Hash options
55
+ def _set_options(inifile, options)
56
+ if inifile
57
+ ini = IniFile.load(inifile)
58
+ raise Exception, 'inifile not found.' if ini.nil?
59
+
60
+ options[:access_key_id] =
61
+ ini['common']['access_key_id'] || ini['s3']['access_key_id']
62
+ options[:secret_access_key] =
63
+ ini['s3']['secret_access_key'] || ini['common']['secret_access_key']
64
+ options[:region] ||=
65
+ ini['s3']['region'] || ini['common']['region']
66
+ end
67
+
68
+ options[:cmd_opts] = {} if options[:cmd_opts].nil?
69
+ options[:cmd_opts][:multipart_threshold] ||= 15_728_640
70
+
71
+ options
72
+ end
73
+ private :_set_options
74
+
75
+ # @private
76
+ # check command_attr
77
+ # @param String command
78
+ # @param Hash options
79
+ # @param String src
80
+ # @param String dest
81
+ def _check_attrs(command, options, src, dest)
82
+ # check region
83
+ raise Exception, 'region is not set.' if options[:region].nil?
84
+
85
+ # check credentials
86
+ if options[:access_key_id].nil?
87
+ raise Exception, 'access_key_id is not set.'
88
+ end
89
+ if options[:secret_access_key].nil?
90
+ raise Exception, 'secret_access_key is not set.'
91
+ end
92
+
93
+ # check command
94
+ raise Exception, 's3 comamnd is not set.' if command.nil?
95
+
96
+ # check command and resources
97
+ case command
98
+ when COMMAND_COPY.to_s
99
+ raise Exception, 'src or dest are not set.' if src.nil? || dest.nil?
100
+
101
+ # check target if file exists
102
+ raise Exception, 'src file does not exist.' unless File.exist?(src)
103
+
104
+ # check if file is not empty
105
+ raise Exception, 'The target file is empty.' unless File.size?(src)
106
+ else
107
+ raise Exception,
108
+ 'command is invalid. '\
109
+ "supported commands are #{S3_COMMANDS.map(&:to_s)}"
110
+ end
111
+ end
112
+ private :_check_attrs
113
+
114
+ # @private
115
+ # copy file(s) between file and s3
116
+ # @param Aws::S3::Client s3_cli
117
+ # @param String src
118
+ # @param String dest
119
+ # @param Hash options
120
+ def _copy(s3_cli, src, dest, options)
121
+ @logger.info "source is #{@src}"
122
+ @logger.info "destination is #{@dest}"
123
+ @logger.info "region is #{options[:region]}"
124
+
125
+ path_info = {}
126
+ # path_info is going to be like:
127
+ # path_info = {
128
+ # "src"=>{
129
+ # "protocol"=>"file",
130
+ # "path"=>"/path/to/file"
131
+ # }
132
+ # "dest"=>{
133
+ # "protocol"=>"s3",
134
+ # "bucket"=>"bucket",
135
+ # "key"=>"test_key"
136
+ # }
137
+ # }
138
+ path_info['src'] = _path_info(src)
139
+ path_info['dest'] = _path_info(dest)
140
+
141
+ mode = _mode_of_copy(path_info)
142
+ @logger.info "copy mode is #{mode}"
143
+
144
+ if mode == MODE_UPLOAD_FROM_LOCAL_TO_S3
145
+ _upload_from_local_to_s3(s3_cli, path_info, options)
146
+ end
147
+ end
148
+ private :_copy
149
+
150
+ # @private
151
+ # get path information
152
+ # @param String path
153
+ # @return Hash {protocol: 'file', path: '/path/to/file'}
154
+ # or {protocol: 's3' or sth,, bucket: 'bucket', key: 'key'}
155
+ def _path_info(path)
156
+ path_info = {}
157
+ path_info['protocol'], bucket_key =
158
+ _divide_protocol_and_bucket_key(path)
159
+
160
+ # path has no protocol
161
+ if path_info['protocol'] == 'file' || bucket_key.nil?
162
+ path_info['path'] = path_info['protocol']
163
+ path_info['protocol'] = 'file'
164
+ # path has a protocol
165
+ elsif bucket_key
166
+ path_info['bucket'], path_info['key'] =
167
+ _divide_bucket_and_key(bucket_key)
168
+ raise Exception, 's3 object key is not set' if path_info['key'].nil?
169
+ else
170
+ raise Exception, 's3 bucket and object key are not set'
171
+ end
172
+
173
+ path_info
174
+ end
175
+ private :_path_info
176
+
177
+ # @private
178
+ # get the mode of copy
179
+ # only MODE_UPLOAD_FROM_LOCAL_TO_S3 is supported now
180
+ # @param Hash path_info
181
+ # @return Symbol
182
+ def _mode_of_copy(path_info)
183
+ # TODO: need to implement
184
+ # MODE_UPLOAD_FROM_LOCAL_TO_S3
185
+ # MODE_UPLOAD_FROM_HDFS_TO_S3
186
+ # MODE_GET_FROM_S3_TO_LOCAL
187
+ # MODE_GET_FROM_S3_TO_HDFS
188
+ # MODE_COPY and
189
+ # MODE_RECURSIVE_UPLOAD_FROM_LOCAL_TO_S3
190
+ # MODE_RECURSIVE_UPLOAD_FROM_HDFS_TO_S3
191
+ # puts path_info['src']['protocol']
192
+ # puts path_info['dest']['protocol']
193
+ # puts S3_PROTOCOLS
194
+ # puts S3_PROTOCOLS.class
195
+ # puts S3_PROTOCOLS.include?(path_info['dest']['protocol'])
196
+
197
+ if path_info['src']['protocol'] == 'file' &&
198
+ S3_PROTOCOLS.include?(path_info['dest']['protocol'])
199
+ return MODE_UPLOAD_FROM_LOCAL_TO_S3
200
+ else
201
+ raise Exception,
202
+ 'only the mode to "copy" from local to s3 is supported now'
203
+ end
204
+ end
205
+ private :_mode_of_copy
206
+
207
+ # @private
208
+ # upload file from local to s3
209
+ # @param Aws::S3::Client s3_cli
210
+ # @param Hash path_info
211
+ # @param Hash options
212
+ def _upload_from_local_to_s3(s3_cli, path_info, options)
213
+ @logger.info 'start uploading...'
214
+ s3_resource = Aws::S3::Resource.new(client: s3_cli)
215
+ obj = s3_resource
216
+ .bucket(path_info['dest']['bucket'])
217
+ .object(path_info['dest']['key'])
218
+
219
+ obj.upload_file(
220
+ path_info['src']['path'],
221
+ options[:cmd_opts].symbolize_keys
222
+ )
223
+ @logger.info 'end upload'
224
+ rescue Aws::S3::MultipartUploadError => errors
225
+ puts "failed upload file to S3: #{errors.message}"
226
+ end
227
+ private :_upload_from_local_to_s3
228
+
229
+ # @private
230
+ # divide path into protocol and s3 bucket + key
231
+ # @param String path
232
+ # @return Array [protocol, bucket + key]
233
+ def _divide_protocol_and_bucket_key(path)
234
+ path.split('://')
235
+ end
236
+ private :_divide_protocol_and_bucket_key
237
+
238
+ # @private
239
+ # divide s3 bucket + key into bucket and key
240
+ # @param String bucket + key
241
+ # @return Array [bucket, key]
242
+ def _divide_bucket_and_key(path)
243
+ path.split('/', 2)
244
+ end
245
+ private :_divide_bucket_and_key
246
+ end
247
+ end
248
+ end
@@ -0,0 +1 @@
1
+ require 'patriot_aws/command/s3'
@@ -0,0 +1,25 @@
1
+ require 'aws-sdk'
2
+
3
+ module PatriotAWS
4
+ module Ext
5
+ module AWS
6
+ def self.included(cls)
7
+ cls.send(:include, Patriot::Util::System)
8
+ end
9
+
10
+ def config_aws(options)
11
+ options.symbolize_keys
12
+ Aws.config.update(
13
+ access_key_id: options[:access_key_id],
14
+ secret_access_key: options[:secret_access_key]
15
+ )
16
+
17
+ if options[:region]
18
+ Aws.config.update(
19
+ region: options[:region]
20
+ )
21
+ end
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1 @@
1
+ require "patriot_aws/ext/aws"
@@ -0,0 +1,4 @@
1
+ class VERSION
2
+ VERSION = "0.1.0"
3
+ PROJECT_NAME = "patriot-aws"
4
+ end
@@ -0,0 +1,2 @@
1
+ require 'patriot_aws/ext'
2
+ require 'patriot_aws/command'
metadata ADDED
@@ -0,0 +1,80 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: patriot-aws
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Takayuki Tanaka
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-04-25 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: aws-sdk
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '2'
27
+ - !ruby/object:Gem::Dependency
28
+ name: patriot-workflow-scheduler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '0.7'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '0.7'
41
+ description: plugins for Patriot Workflow Scheduler, which deal with AWS such as S3.
42
+ email:
43
+ - takat007@gmail.com
44
+ executables: []
45
+ extensions: []
46
+ extra_rdoc_files: []
47
+ files:
48
+ - init.rb
49
+ - lib/patriot_aws.rb
50
+ - lib/patriot_aws/command.rb
51
+ - lib/patriot_aws/command/s3.rb
52
+ - lib/patriot_aws/ext.rb
53
+ - lib/patriot_aws/ext/aws.rb
54
+ - lib/patriot_aws/version.rb
55
+ homepage: https://github.com/CyberAgent/patriot-workflow-scheduler/tree/master/plugins/patriot-aws
56
+ licenses:
57
+ - Apache-2.0
58
+ metadata: {}
59
+ post_install_message:
60
+ rdoc_options: []
61
+ require_paths:
62
+ - lib
63
+ required_ruby_version: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ! '>='
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ required_rubygems_version: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - ! '>='
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ requirements: []
74
+ rubyforge_project: patriot-aws
75
+ rubygems_version: 2.4.7
76
+ signing_key:
77
+ specification_version: 4
78
+ summary: AWS plugin for Patriot Workflow Scheduler
79
+ test_files: []
80
+ has_rdoc: