patriot-aws 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/init.rb +2 -0
- data/lib/patriot_aws/command/s3.rb +248 -0
- data/lib/patriot_aws/command.rb +1 -0
- data/lib/patriot_aws/ext/aws.rb +25 -0
- data/lib/patriot_aws/ext.rb +1 -0
- data/lib/patriot_aws/version.rb +4 -0
- data/lib/patriot_aws.rb +2 -0
- metadata +80 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
YTA4NTk4NTg3NjFjZGQ2NTc3OTBkYWQyNzViMjcyNTkzYzFiZDZhMA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZGMwYmJhNDkwZGIwYzhlYjdjZjM2NTQwZjNlNDljNmUyMDBmMzVlYg==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MTA2NGJhMDEyNDNjNzhlOGQ1ZTY5MTI5Mzc5Y2FjZjBjNzljNzdmNmY2Zjc1
|
10
|
+
ODBlZmZjMDM1OWIxNDQxZjhlOGM0YzEyM2IzZWJiOGE1YjRmNTdhMzA2OGEx
|
11
|
+
NjQzZTdiMWI1YWYzM2EyODcwODg1ZTdlNWZkNGM4ZGMyNzJhNWE=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ODZmNTI4OWY4OWM1MzdjMDI0YzM0MWRhYjhmYWEwMmM4MGFhMjc5NTllMWRm
|
14
|
+
ODRhOTE2Zjg4NDNiYjY1NDY1NTg0ZTMwMWU2OTMyYmU1N2QxMTMzNjBkOGIz
|
15
|
+
OTkyOTU4ODgxMjBhNWNkMGM5OWY1MDhjZDAzZTM2OTkyZjRkMjE=
|
data/init.rb
ADDED
@@ -0,0 +1,248 @@
|
|
1
|
+
module PatriotAWS
|
2
|
+
module Command
|
3
|
+
class S3Command < Patriot::Command::Base
|
4
|
+
declare_command_name :s3
|
5
|
+
include PatriotAWS::Ext::AWS
|
6
|
+
|
7
|
+
command_attr :name, :name_suffix, :inifile,
|
8
|
+
:command, :src, :dest, :options
|
9
|
+
|
10
|
+
MODE_UPLOAD_FROM_LOCAL_TO_S3 = :mode_upload_from_local_to_s3
|
11
|
+
S3_PROTOCOLS = %w(s3 s3n s3a).freeze
|
12
|
+
COMMAND_COPY = :copy
|
13
|
+
S3_COMMANDS = [COMMAND_COPY].freeze
|
14
|
+
|
15
|
+
def job_id
|
16
|
+
job_id = "#{command_name}_#{@command}_#{@name}_#{@name_suffix}"
|
17
|
+
job_id
|
18
|
+
end
|
19
|
+
|
20
|
+
# @see Patriot::Command::Base#configure
|
21
|
+
def configure
|
22
|
+
@name_suffix ||= _date_
|
23
|
+
self
|
24
|
+
end
|
25
|
+
|
26
|
+
def execute
|
27
|
+
@logger.info "start s3 #{@command}"
|
28
|
+
|
29
|
+
@options ||= {}
|
30
|
+
@options = @options.symbolize_keys
|
31
|
+
@options = _set_options(@inifile, @options)
|
32
|
+
|
33
|
+
_check_attrs(@command, @options, @src, @dest)
|
34
|
+
|
35
|
+
config_aws(@options)
|
36
|
+
s3_cli = Aws::S3::Client.new
|
37
|
+
|
38
|
+
case @command
|
39
|
+
when COMMAND_COPY.to_s
|
40
|
+
_copy(s3_cli, @src, @dest, @options)
|
41
|
+
else
|
42
|
+
# should not reach here because this check is already done
|
43
|
+
# in configure
|
44
|
+
raise Exception,
|
45
|
+
'command is invalid. '\
|
46
|
+
"supported commands are #{S3_COMMANDS.map(&:to_s)}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
# @private
|
51
|
+
# get inifile info and set options parameters
|
52
|
+
# @param String inifile
|
53
|
+
# @param Hash options
|
54
|
+
# @return Hash options
|
55
|
+
def _set_options(inifile, options)
|
56
|
+
if inifile
|
57
|
+
ini = IniFile.load(inifile)
|
58
|
+
raise Exception, 'inifile not found.' if ini.nil?
|
59
|
+
|
60
|
+
options[:access_key_id] =
|
61
|
+
ini['common']['access_key_id'] || ini['s3']['access_key_id']
|
62
|
+
options[:secret_access_key] =
|
63
|
+
ini['s3']['secret_access_key'] || ini['common']['secret_access_key']
|
64
|
+
options[:region] ||=
|
65
|
+
ini['s3']['region'] || ini['common']['region']
|
66
|
+
end
|
67
|
+
|
68
|
+
options[:cmd_opts] = {} if options[:cmd_opts].nil?
|
69
|
+
options[:cmd_opts][:multipart_threshold] ||= 15_728_640
|
70
|
+
|
71
|
+
options
|
72
|
+
end
|
73
|
+
private :_set_options
|
74
|
+
|
75
|
+
# @private
|
76
|
+
# check command_attr
|
77
|
+
# @param String command
|
78
|
+
# @param Hash options
|
79
|
+
# @param String src
|
80
|
+
# @param String dest
|
81
|
+
def _check_attrs(command, options, src, dest)
|
82
|
+
# check region
|
83
|
+
raise Exception, 'region is not set.' if options[:region].nil?
|
84
|
+
|
85
|
+
# check credentials
|
86
|
+
if options[:access_key_id].nil?
|
87
|
+
raise Exception, 'access_key_id is not set.'
|
88
|
+
end
|
89
|
+
if options[:secret_access_key].nil?
|
90
|
+
raise Exception, 'secret_access_key is not set.'
|
91
|
+
end
|
92
|
+
|
93
|
+
# check command
|
94
|
+
raise Exception, 's3 comamnd is not set.' if command.nil?
|
95
|
+
|
96
|
+
# check command and resources
|
97
|
+
case command
|
98
|
+
when COMMAND_COPY.to_s
|
99
|
+
raise Exception, 'src or dest are not set.' if src.nil? || dest.nil?
|
100
|
+
|
101
|
+
# check target if file exists
|
102
|
+
raise Exception, 'src file does not exist.' unless File.exist?(src)
|
103
|
+
|
104
|
+
# check if file is not empty
|
105
|
+
raise Exception, 'The target file is empty.' unless File.size?(src)
|
106
|
+
else
|
107
|
+
raise Exception,
|
108
|
+
'command is invalid. '\
|
109
|
+
"supported commands are #{S3_COMMANDS.map(&:to_s)}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
private :_check_attrs
|
113
|
+
|
114
|
+
# @private
|
115
|
+
# copy file(s) between file and s3
|
116
|
+
# @param Aws::S3::Client s3_cli
|
117
|
+
# @param String src
|
118
|
+
# @param String dest
|
119
|
+
# @param Hash options
|
120
|
+
def _copy(s3_cli, src, dest, options)
|
121
|
+
@logger.info "source is #{@src}"
|
122
|
+
@logger.info "destination is #{@dest}"
|
123
|
+
@logger.info "region is #{options[:region]}"
|
124
|
+
|
125
|
+
path_info = {}
|
126
|
+
# path_info is going to be like:
|
127
|
+
# path_info = {
|
128
|
+
# "src"=>{
|
129
|
+
# "protocol"=>"file",
|
130
|
+
# "path"=>"/path/to/file"
|
131
|
+
# }
|
132
|
+
# "dest"=>{
|
133
|
+
# "protocol"=>"s3",
|
134
|
+
# "bucket"=>"bucket",
|
135
|
+
# "key"=>"test_key"
|
136
|
+
# }
|
137
|
+
# }
|
138
|
+
path_info['src'] = _path_info(src)
|
139
|
+
path_info['dest'] = _path_info(dest)
|
140
|
+
|
141
|
+
mode = _mode_of_copy(path_info)
|
142
|
+
@logger.info "copy mode is #{mode}"
|
143
|
+
|
144
|
+
if mode == MODE_UPLOAD_FROM_LOCAL_TO_S3
|
145
|
+
_upload_from_local_to_s3(s3_cli, path_info, options)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
private :_copy
|
149
|
+
|
150
|
+
# @private
|
151
|
+
# get path information
|
152
|
+
# @param String path
|
153
|
+
# @return Hash {protocol: 'file', path: '/path/to/file'}
|
154
|
+
# or {protocol: 's3' or sth,, bucket: 'bucket', key: 'key'}
|
155
|
+
def _path_info(path)
|
156
|
+
path_info = {}
|
157
|
+
path_info['protocol'], bucket_key =
|
158
|
+
_divide_protocol_and_bucket_key(path)
|
159
|
+
|
160
|
+
# path has no protocol
|
161
|
+
if path_info['protocol'] == 'file' || bucket_key.nil?
|
162
|
+
path_info['path'] = path_info['protocol']
|
163
|
+
path_info['protocol'] = 'file'
|
164
|
+
# path has a protocol
|
165
|
+
elsif bucket_key
|
166
|
+
path_info['bucket'], path_info['key'] =
|
167
|
+
_divide_bucket_and_key(bucket_key)
|
168
|
+
raise Exception, 's3 object key is not set' if path_info['key'].nil?
|
169
|
+
else
|
170
|
+
raise Exception, 's3 bucket and object key are not set'
|
171
|
+
end
|
172
|
+
|
173
|
+
path_info
|
174
|
+
end
|
175
|
+
private :_path_info
|
176
|
+
|
177
|
+
# @private
|
178
|
+
# get the mode of copy
|
179
|
+
# only MODE_UPLOAD_FROM_LOCAL_TO_S3 is supported now
|
180
|
+
# @param Hash path_info
|
181
|
+
# @return Symbol
|
182
|
+
def _mode_of_copy(path_info)
|
183
|
+
# TODO: need to implement
|
184
|
+
# MODE_UPLOAD_FROM_LOCAL_TO_S3
|
185
|
+
# MODE_UPLOAD_FROM_HDFS_TO_S3
|
186
|
+
# MODE_GET_FROM_S3_TO_LOCAL
|
187
|
+
# MODE_GET_FROM_S3_TO_HDFS
|
188
|
+
# MODE_COPY and
|
189
|
+
# MODE_RECURSIVE_UPLOAD_FROM_LOCAL_TO_S3
|
190
|
+
# MODE_RECURSIVE_UPLOAD_FROM_HDFS_TO_S3
|
191
|
+
# puts path_info['src']['protocol']
|
192
|
+
# puts path_info['dest']['protocol']
|
193
|
+
# puts S3_PROTOCOLS
|
194
|
+
# puts S3_PROTOCOLS.class
|
195
|
+
# puts S3_PROTOCOLS.include?(path_info['dest']['protocol'])
|
196
|
+
|
197
|
+
if path_info['src']['protocol'] == 'file' &&
|
198
|
+
S3_PROTOCOLS.include?(path_info['dest']['protocol'])
|
199
|
+
return MODE_UPLOAD_FROM_LOCAL_TO_S3
|
200
|
+
else
|
201
|
+
raise Exception,
|
202
|
+
'only the mode to "copy" from local to s3 is supported now'
|
203
|
+
end
|
204
|
+
end
|
205
|
+
private :_mode_of_copy
|
206
|
+
|
207
|
+
# @private
|
208
|
+
# upload file from local to s3
|
209
|
+
# @param Aws::S3::Client s3_cli
|
210
|
+
# @param Hash path_info
|
211
|
+
# @param Hash options
|
212
|
+
def _upload_from_local_to_s3(s3_cli, path_info, options)
|
213
|
+
@logger.info 'start uploading...'
|
214
|
+
s3_resource = Aws::S3::Resource.new(client: s3_cli)
|
215
|
+
obj = s3_resource
|
216
|
+
.bucket(path_info['dest']['bucket'])
|
217
|
+
.object(path_info['dest']['key'])
|
218
|
+
|
219
|
+
obj.upload_file(
|
220
|
+
path_info['src']['path'],
|
221
|
+
options[:cmd_opts].symbolize_keys
|
222
|
+
)
|
223
|
+
@logger.info 'end upload'
|
224
|
+
rescue Aws::S3::MultipartUploadError => errors
|
225
|
+
puts "failed upload file to S3: #{errors.message}"
|
226
|
+
end
|
227
|
+
private :_upload_from_local_to_s3
|
228
|
+
|
229
|
+
# @private
|
230
|
+
# divide path into protocol and s3 bucket + key
|
231
|
+
# @param String path
|
232
|
+
# @return Array [protocol, bucket + key]
|
233
|
+
def _divide_protocol_and_bucket_key(path)
|
234
|
+
path.split('://')
|
235
|
+
end
|
236
|
+
private :_divide_protocol_and_bucket_key
|
237
|
+
|
238
|
+
# @private
|
239
|
+
# divide s3 bucket + key into bucket and key
|
240
|
+
# @param String bucket + key
|
241
|
+
# @return Array [bucket, key]
|
242
|
+
def _divide_bucket_and_key(path)
|
243
|
+
path.split('/', 2)
|
244
|
+
end
|
245
|
+
private :_divide_bucket_and_key
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require 'patriot_aws/command/s3'
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module PatriotAWS
|
4
|
+
module Ext
|
5
|
+
module AWS
|
6
|
+
def self.included(cls)
|
7
|
+
cls.send(:include, Patriot::Util::System)
|
8
|
+
end
|
9
|
+
|
10
|
+
def config_aws(options)
|
11
|
+
options.symbolize_keys
|
12
|
+
Aws.config.update(
|
13
|
+
access_key_id: options[:access_key_id],
|
14
|
+
secret_access_key: options[:secret_access_key]
|
15
|
+
)
|
16
|
+
|
17
|
+
if options[:region]
|
18
|
+
Aws.config.update(
|
19
|
+
region: options[:region]
|
20
|
+
)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
require "patriot_aws/ext/aws"
|
data/lib/patriot_aws.rb
ADDED
metadata
ADDED
@@ -0,0 +1,80 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: patriot-aws
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Takayuki Tanaka
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-04-25 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: patriot-workflow-scheduler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.7'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.7'
|
41
|
+
description: plugins for Patriot Workflow Scheduler, which deal with AWS such as S3.
|
42
|
+
email:
|
43
|
+
- takat007@gmail.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- init.rb
|
49
|
+
- lib/patriot_aws.rb
|
50
|
+
- lib/patriot_aws/command.rb
|
51
|
+
- lib/patriot_aws/command/s3.rb
|
52
|
+
- lib/patriot_aws/ext.rb
|
53
|
+
- lib/patriot_aws/ext/aws.rb
|
54
|
+
- lib/patriot_aws/version.rb
|
55
|
+
homepage: https://github.com/CyberAgent/patriot-workflow-scheduler/tree/master/plugins/patriot-aws
|
56
|
+
licenses:
|
57
|
+
- Apache-2.0
|
58
|
+
metadata: {}
|
59
|
+
post_install_message:
|
60
|
+
rdoc_options: []
|
61
|
+
require_paths:
|
62
|
+
- lib
|
63
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ! '>='
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ! '>='
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '0'
|
73
|
+
requirements: []
|
74
|
+
rubyforge_project: patriot-aws
|
75
|
+
rubygems_version: 2.4.7
|
76
|
+
signing_key:
|
77
|
+
specification_version: 4
|
78
|
+
summary: AWS plugin for Patriot Workflow Scheduler
|
79
|
+
test_files: []
|
80
|
+
has_rdoc:
|