elasticity 2.4 → 2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.md +5 -0
- data/README.md +15 -8
- data/elasticity.gemspec +1 -1
- data/lib/elasticity.rb +0 -2
- data/lib/elasticity/aws_request.rb +1 -1
- data/lib/elasticity/hive_step.rb +6 -6
- data/lib/elasticity/instance_group.rb +12 -4
- data/lib/elasticity/job_flow.rb +16 -6
- data/lib/elasticity/sync_to_s3.rb +20 -10
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/aws_request_spec.rb +1 -1
- data/spec/lib/elasticity/hive_step_spec.rb +14 -10
- data/spec/lib/elasticity/job_flow_integration_spec.rb +5 -1
- data/spec/lib/elasticity/sync_to_s3_spec.rb +70 -58
- metadata +8 -11
- data/lib/elasticity/support/conditional_raise.rb +0 -23
- data/spec/lib/elasticity/support/conditional_raise_spec.rb +0 -35
data/HISTORY.md
CHANGED
data/README.md
CHANGED
@@ -16,7 +16,7 @@ gem install elasticity
|
|
16
16
|
or in your Gemfile
|
17
17
|
|
18
18
|
```
|
19
|
-
gem 'elasticity', '~> 2.
|
19
|
+
gem 'elasticity', '~> 2.5'
|
20
20
|
```
|
21
21
|
|
22
22
|
This will ensure that you protect yourself from API changes, which will only be made in major revisions.
|
@@ -273,20 +273,27 @@ jobflow.add_step(jar_step)
|
|
273
273
|
|
274
274
|
## 6 - Upload Assets (optional)
|
275
275
|
|
276
|
-
This isn't part of ```JobFlow```; more of an aside
|
276
|
+
This isn't part of ```JobFlow```; more of an aside. Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. Most commonly this will be a set of resources to run the job (e.g. JAR files, streaming scripts, etc.) and a set of resources used by the job itself (e.g. a TSV file with a range of valid values, join tables, etc.).
|
277
277
|
|
278
278
|
```ruby
|
279
|
-
# Specify the bucket
|
280
|
-
s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret')
|
279
|
+
# Specify the bucket name, AWS credentials and region
|
280
|
+
s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret', 'region')
|
281
281
|
|
282
|
-
#
|
282
|
+
# Alternatively, specify nothing :)
|
283
|
+
# - Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
|
284
|
+
# - Use the 'us-east-1' region by default
|
283
285
|
# s3 = Elasticity::SyncToS3('my-bucket')
|
284
286
|
|
285
|
-
# Recursively sync the contents of '/
|
286
|
-
s3.sync('/
|
287
|
+
# Recursively sync the contents of '/foo' under the remote location 'remote-dir/this-job'
|
288
|
+
s3.sync('/foo', 'remote-dir/this-job')
|
289
|
+
|
290
|
+
# Sync a single file to a remote directory
|
291
|
+
s3.sync('/foo/this-job/tables/join.tsv', 'remote-dir/this-job/tables')
|
287
292
|
```
|
288
293
|
|
289
|
-
If the
|
294
|
+
If the bucket doesn't exist, it will be created.
|
295
|
+
|
296
|
+
If a file already exists, there is an MD5 checksum evaluation. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/tables/join.tsv``` in your EMR jobs.
|
290
297
|
|
291
298
|
## 7 - Run the Job Flow
|
292
299
|
|
data/elasticity.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.add_dependency('nokogiri')
|
16
16
|
s.add_dependency('fog')
|
17
17
|
|
18
|
-
s.add_development_dependency('rake')
|
18
|
+
s.add_development_dependency('rake', '~> 0.9')
|
19
19
|
s.add_development_dependency('rspec', '~> 2.11.0')
|
20
20
|
s.add_development_dependency('timecop')
|
21
21
|
s.add_development_dependency('fakefs', '~> 0.4')
|
data/lib/elasticity.rb
CHANGED
@@ -50,7 +50,7 @@ module Elasticity
|
|
50
50
|
def get_secret_key(secret)
|
51
51
|
return secret if secret
|
52
52
|
return ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
|
53
|
-
raise MissingKeyError, 'Please provide a secret key or set
|
53
|
+
raise MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.'
|
54
54
|
end
|
55
55
|
|
56
56
|
# (Used from RightScale's right_aws gem.)
|
data/lib/elasticity/hive_step.rb
CHANGED
@@ -17,7 +17,7 @@ module Elasticity
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def to_aws_step(job_flow)
|
20
|
-
args = %w(s3://elasticmapreduce/libs/hive/hive-script --run-hive-script --args)
|
20
|
+
args = %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions latest --run-hive-script --args)
|
21
21
|
args.concat(['-f', @script])
|
22
22
|
@variables.keys.sort.each do |name|
|
23
23
|
args.concat(['-d', "#{name}=#{@variables[name]}"])
|
@@ -43,11 +43,11 @@ module Elasticity
|
|
43
43
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
44
44
|
:args => [
|
45
45
|
's3://elasticmapreduce/libs/hive/hive-script',
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
46
|
+
'--base-path',
|
47
|
+
's3://elasticmapreduce/libs/hive/',
|
48
|
+
'--install-hive',
|
49
|
+
'--hive-versions',
|
50
|
+
'latest'
|
51
51
|
],
|
52
52
|
},
|
53
53
|
:name => 'Elasticity - Install Hive'
|
@@ -19,19 +19,27 @@ module Elasticity
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def count=(instance_count)
|
22
|
-
|
23
|
-
|
22
|
+
if instance_count <= 0
|
23
|
+
raise ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
|
24
|
+
end
|
25
|
+
if @role == 'MASTER' && instance_count != 1
|
26
|
+
raise ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
|
27
|
+
end
|
24
28
|
@count = instance_count
|
25
29
|
end
|
26
30
|
|
27
31
|
def role=(group_role)
|
28
|
-
|
32
|
+
if !ROLES.include?(group_role)
|
33
|
+
raise ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
|
34
|
+
end
|
29
35
|
@count = 1 if group_role == 'MASTER'
|
30
36
|
@role = group_role
|
31
37
|
end
|
32
38
|
|
33
39
|
def set_spot_instances(bid_price)
|
34
|
-
|
40
|
+
if bid_price < 0
|
41
|
+
raise ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
|
42
|
+
end
|
35
43
|
@bid_price = bid_price
|
36
44
|
@market = 'SPOT'
|
37
45
|
end
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -21,7 +21,7 @@ module Elasticity
|
|
21
21
|
|
22
22
|
attr_reader :access_key
|
23
23
|
attr_reader :secret_key
|
24
|
-
|
24
|
+
|
25
25
|
def initialize(access=nil, secret=nil)
|
26
26
|
@action_on_failure = 'TERMINATE_JOB_FLOW'
|
27
27
|
@hadoop_version = '1.0.3'
|
@@ -73,7 +73,9 @@ module Elasticity
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def add_bootstrap_action(bootstrap_action)
|
76
|
-
|
76
|
+
if is_jobflow_running?
|
77
|
+
raise JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
|
78
|
+
end
|
77
79
|
@bootstrap_actions << bootstrap_action
|
78
80
|
end
|
79
81
|
|
@@ -106,18 +108,26 @@ module Elasticity
|
|
106
108
|
end
|
107
109
|
|
108
110
|
def run
|
109
|
-
|
110
|
-
|
111
|
+
if @jobflow_steps.empty?
|
112
|
+
raise JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
|
113
|
+
end
|
114
|
+
if is_jobflow_running?
|
115
|
+
raise JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
|
116
|
+
end
|
111
117
|
@jobflow_id = emr.run_job_flow(jobflow_config)
|
112
118
|
end
|
113
119
|
|
114
120
|
def shutdown
|
115
|
-
|
121
|
+
if !is_jobflow_running?
|
122
|
+
raise JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
|
123
|
+
end
|
116
124
|
emr.terminate_jobflows(@jobflow_id)
|
117
125
|
end
|
118
126
|
|
119
127
|
def status
|
120
|
-
|
128
|
+
if !is_jobflow_running?
|
129
|
+
raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
|
130
|
+
end
|
121
131
|
emr.describe_jobflow(@jobflow_id)
|
122
132
|
end
|
123
133
|
|
@@ -2,38 +2,45 @@ module Elasticity
|
|
2
2
|
|
3
3
|
class NoBucketError < StandardError; end
|
4
4
|
class NoDirectoryError < StandardError; end
|
5
|
+
class NoFileError < StandardError; end
|
5
6
|
|
6
7
|
class SyncToS3
|
7
8
|
|
8
9
|
attr_reader :access_key
|
9
10
|
attr_reader :secret_key
|
10
11
|
attr_reader :bucket_name
|
12
|
+
attr_reader :region
|
11
13
|
|
12
|
-
def initialize(bucket, access=nil, secret=nil)
|
14
|
+
def initialize(bucket, access=nil, secret=nil, region=nil)
|
13
15
|
@access_key = get_access_key(access)
|
14
16
|
@secret_key = get_secret_key(secret)
|
15
17
|
@bucket_name = bucket
|
18
|
+
@region = region ||= 'us-east-1'
|
16
19
|
end
|
17
20
|
|
18
21
|
def sync(local, remote)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
if File.directory?(local)
|
23
|
+
sync_dir(local, remote)
|
24
|
+
else
|
25
|
+
sync_file(local, remote)
|
26
|
+
end
|
22
27
|
end
|
23
28
|
|
24
29
|
private
|
25
30
|
|
26
|
-
def sync_dir(
|
27
|
-
|
31
|
+
def sync_dir(dir_name, remote_dir)
|
32
|
+
raise NoDirectoryError, "Directory '#{dir_name}' does not exist or is not a directory" unless File.directory?(dir_name)
|
33
|
+
Dir.glob(File.join([dir_name, '*'])).each do |entry|
|
28
34
|
if File.directory?(entry)
|
29
|
-
sync_dir(entry, [
|
35
|
+
sync_dir(entry, [remote_dir, File.basename(entry)].join('/'))
|
30
36
|
else
|
31
|
-
sync_file(entry,
|
37
|
+
sync_file(entry, remote_dir)
|
32
38
|
end
|
33
39
|
end
|
34
40
|
end
|
35
41
|
|
36
42
|
def sync_file(file_name, remote_dir)
|
43
|
+
raise NoFileError, "File '#{file_name}' does not exist" unless File.exists?(file_name)
|
37
44
|
remote_dir = remote_dir.gsub(/^(\/)/, '')
|
38
45
|
remote_path = (remote_dir.empty?) ? (File.basename(file_name)) : [remote_dir, File.basename(file_name)].join('/')
|
39
46
|
metadata = bucket.files.head(remote_path)
|
@@ -46,14 +53,17 @@ module Elasticity
|
|
46
53
|
})
|
47
54
|
end
|
48
55
|
|
56
|
+
private
|
57
|
+
|
49
58
|
def bucket
|
50
|
-
|
51
|
-
@bucket ||=
|
59
|
+
@bucket ||= s3.directories.find { |d| d.key == @bucket_name }
|
60
|
+
@bucket ||= s3.directories.create(:key => @bucket_name)
|
52
61
|
end
|
53
62
|
|
54
63
|
def s3
|
55
64
|
@connection ||= Fog::Storage.new({
|
56
65
|
:provider => 'AWS',
|
66
|
+
:region => @region,
|
57
67
|
:aws_access_key_id => @access_key,
|
58
68
|
:aws_secret_access_key => @secret_key
|
59
69
|
})
|
data/lib/elasticity/version.rb
CHANGED
@@ -68,7 +68,7 @@ describe Elasticity::AwsRequest do
|
|
68
68
|
it 'should raise an error' do
|
69
69
|
expect {
|
70
70
|
missing_something.access_key
|
71
|
-
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set
|
71
|
+
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
@@ -20,10 +20,14 @@ describe Elasticity::HiveStep do
|
|
20
20
|
step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
|
21
21
|
step[:hadoop_jar_step][:args].should start_with([
|
22
22
|
's3://elasticmapreduce/libs/hive/hive-script',
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
'--base-path',
|
24
|
+
's3://elasticmapreduce/libs/hive/',
|
25
|
+
'--hive-versions',
|
26
|
+
'latest',
|
27
|
+
'--run-hive-script',
|
28
|
+
'--args',
|
29
|
+
'-f',
|
30
|
+
'script.hql'
|
27
31
|
])
|
28
32
|
end
|
29
33
|
|
@@ -39,7 +43,7 @@ describe Elasticity::HiveStep do
|
|
39
43
|
|
40
44
|
it 'should convert to aws step format' do
|
41
45
|
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
|
42
|
-
step[:hadoop_jar_step][:args][
|
46
|
+
step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
|
43
47
|
end
|
44
48
|
end
|
45
49
|
|
@@ -60,11 +64,11 @@ describe Elasticity::HiveStep do
|
|
60
64
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
61
65
|
:args => [
|
62
66
|
's3://elasticmapreduce/libs/hive/hive-script',
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
67
|
+
'--base-path',
|
68
|
+
's3://elasticmapreduce/libs/hive/',
|
69
|
+
'--install-hive',
|
70
|
+
'--hive-versions',
|
71
|
+
'latest'
|
68
72
|
],
|
69
73
|
},
|
70
74
|
:name => 'Elasticity - Install Hive'
|
@@ -68,7 +68,11 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
68
68
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
69
69
|
:args => [
|
70
70
|
's3://elasticmapreduce/libs/hive/hive-script',
|
71
|
-
|
71
|
+
'--base-path',
|
72
|
+
's3://elasticmapreduce/libs/hive/',
|
73
|
+
'--hive-versions',
|
74
|
+
'latest',
|
75
|
+
'--run-hive-script',
|
72
76
|
'--args',
|
73
77
|
'-f', 's3n://slif-hive/test.q',
|
74
78
|
'-d', 'OUTPUT=s3n://slif-test/output'
|
@@ -17,18 +17,19 @@ describe Elasticity::SyncToS3 do
|
|
17
17
|
describe 'basic assignment' do
|
18
18
|
|
19
19
|
it 'should set the proper values' do
|
20
|
-
sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret')
|
20
|
+
sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret', 'region')
|
21
21
|
sync.access_key.should == 'access'
|
22
22
|
sync.secret_key.should == 'secret'
|
23
23
|
sync.bucket_name.should == 'bucket'
|
24
|
+
sync.region.should == 'region'
|
24
25
|
end
|
25
26
|
|
26
27
|
end
|
27
28
|
|
28
|
-
context 'when
|
29
|
+
context 'when parameters are missing' do
|
29
30
|
|
30
|
-
let(:
|
31
|
-
let(:
|
31
|
+
let(:optional_params_nil) { Elasticity::SyncToS3.new('_', nil, nil, nil) }
|
32
|
+
let(:optional_params_missing) { Elasticity::SyncToS3.new('_') }
|
32
33
|
|
33
34
|
before do
|
34
35
|
ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return(access_key)
|
@@ -38,13 +39,21 @@ describe Elasticity::SyncToS3 do
|
|
38
39
|
context 'when environment variables are present' do
|
39
40
|
let(:access_key) { 'ENV_ACCESS' }
|
40
41
|
let(:secret_key) { 'ENV_SECRET' }
|
41
|
-
it 'should assign them to the keys' do
|
42
|
-
both_keys_nil.access_key.should == 'ENV_ACCESS'
|
43
|
-
both_keys_nil.secret_key.should == 'ENV_SECRET'
|
44
42
|
|
45
|
-
|
46
|
-
|
43
|
+
it 'should assign environment variables to the keys' do
|
44
|
+
optional_params_nil.access_key.should == 'ENV_ACCESS'
|
45
|
+
optional_params_nil.secret_key.should == 'ENV_SECRET'
|
46
|
+
optional_params_missing.access_key.should == 'ENV_ACCESS'
|
47
|
+
optional_params_missing.secret_key.should == 'ENV_SECRET'
|
47
48
|
end
|
49
|
+
|
50
|
+
it 'should set the region to us-east-1' do
|
51
|
+
optional_params_nil.region.should == 'us-east-1'
|
52
|
+
optional_params_nil.region.should == 'us-east-1'
|
53
|
+
optional_params_missing.region.should == 'us-east-1'
|
54
|
+
optional_params_missing.region.should == 'us-east-1'
|
55
|
+
end
|
56
|
+
|
48
57
|
end
|
49
58
|
|
50
59
|
context 'when environment variables are not present' do
|
@@ -54,7 +63,7 @@ describe Elasticity::SyncToS3 do
|
|
54
63
|
let(:secret_key) { '_' }
|
55
64
|
it 'should raise an error' do
|
56
65
|
expect {
|
57
|
-
|
66
|
+
optional_params_nil # Trigger instantiation
|
58
67
|
}.to raise_error(Elasticity::MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.')
|
59
68
|
end
|
60
69
|
end
|
@@ -64,7 +73,7 @@ describe Elasticity::SyncToS3 do
|
|
64
73
|
let(:secret_key) { nil }
|
65
74
|
it 'should raise an error' do
|
66
75
|
expect {
|
67
|
-
|
76
|
+
optional_params_nil # Trigger instantiation
|
68
77
|
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
|
69
78
|
end
|
70
79
|
end
|
@@ -77,50 +86,23 @@ describe Elasticity::SyncToS3 do
|
|
77
86
|
|
78
87
|
describe '#sync' do
|
79
88
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
s3.directories.create(:key => bucket_name)
|
84
|
-
end
|
85
|
-
|
86
|
-
context 'when the local directory exists' do
|
87
|
-
before do
|
88
|
-
FileUtils.mkdir('GOOD_DIR')
|
89
|
-
end
|
90
|
-
it 'should sync that directory' do
|
91
|
-
sync_to_s3.should_receive(:sync_dir).with('GOOD_DIR', 'REMOTE_DIR')
|
92
|
-
sync_to_s3.sync('GOOD_DIR', 'REMOTE_DIR')
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
context 'when the local directory does not exist' do
|
97
|
-
it 'should raise an error' do
|
98
|
-
expect {
|
99
|
-
sync_to_s3.sync('BAD_DIR', '_')
|
100
|
-
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'BAD_DIR' does not exist or is not a directory")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
context 'when the local directory is not a directory' do
|
105
|
-
before do
|
106
|
-
FileUtils.touch('NOT_A_DIR')
|
107
|
-
end
|
108
|
-
it 'should raise an error' do
|
109
|
-
expect {
|
110
|
-
sync_to_s3.sync('NOT_A_DIR', '_')
|
111
|
-
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
|
112
|
-
end
|
113
|
-
end
|
89
|
+
let(:dir_name) { 'GOOD_DIR' }
|
90
|
+
let(:file_name) { File.join(%w(GOOD_DIR file_1)) }
|
91
|
+
let(:remote_dir) { 'REMOTE_DIR' }
|
114
92
|
|
93
|
+
before do
|
94
|
+
FileUtils.mkdir(dir_name)
|
95
|
+
FileUtils.touch(file_name)
|
96
|
+
end
|
97
|
+
|
98
|
+
it 'should sync directories' do
|
99
|
+
sync_to_s3.should_receive(:sync_dir).with(dir_name, remote_dir)
|
100
|
+
sync_to_s3.sync(dir_name, remote_dir)
|
115
101
|
end
|
116
102
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
expect {
|
121
|
-
sync_to_s3.sync('_', '_')
|
122
|
-
}.to raise_error(Elasticity::NoBucketError, "Bucket 'BAD_BUCKET' does not exist")
|
123
|
-
end
|
103
|
+
it 'should sync files' do
|
104
|
+
sync_to_s3.should_receive(:sync_file).with(file_name, remote_dir)
|
105
|
+
sync_to_s3.sync(file_name, remote_dir)
|
124
106
|
end
|
125
107
|
|
126
108
|
end
|
@@ -156,6 +138,14 @@ describe Elasticity::SyncToS3 do
|
|
156
138
|
end
|
157
139
|
end
|
158
140
|
|
141
|
+
context 'when the directory does not exist or is not a directory' do
|
142
|
+
it 'should raise an error' do
|
143
|
+
expect {
|
144
|
+
sync_to_s3.send(:sync_dir, 'NOT_A_DIR', '_')
|
145
|
+
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
159
149
|
end
|
160
150
|
|
161
151
|
describe '#sync_file' do
|
@@ -164,13 +154,13 @@ describe Elasticity::SyncToS3 do
|
|
164
154
|
let(:file_name) { 'test.out' }
|
165
155
|
let(:full_path) { File.join([local_dir, file_name]) }
|
166
156
|
let(:remote_dir) { 'job/assets' }
|
167
|
-
let(:remote_path) { "#{remote_dir}/#{file_name}"}
|
157
|
+
let(:remote_path) { "#{remote_dir}/#{file_name}" }
|
168
158
|
let(:file_data) { 'Some test content' }
|
169
159
|
|
170
160
|
before do
|
171
161
|
s3.directories.create(:key => bucket_name)
|
172
162
|
FileUtils.makedirs(local_dir)
|
173
|
-
File.open(full_path, 'w') {|f| f.write(file_data) }
|
163
|
+
File.open(full_path, 'w') { |f| f.write(file_data) }
|
174
164
|
end
|
175
165
|
|
176
166
|
it 'should write the specified file into the remote directory' do
|
@@ -202,34 +192,56 @@ describe Elasticity::SyncToS3 do
|
|
202
192
|
end
|
203
193
|
|
204
194
|
context 'when remote dir is empty' do
|
205
|
-
let(:remote_dir) {''}
|
195
|
+
let(:remote_dir) { '' }
|
206
196
|
it 'should place files in the root without a bunk empty folder name' do
|
207
197
|
s3.directories[0].files.head(file_name).should_not be_nil
|
208
198
|
end
|
209
199
|
end
|
210
200
|
|
211
201
|
context 'when remote dir is /' do
|
212
|
-
let(:remote_dir) {'/'}
|
202
|
+
let(:remote_dir) { '/' }
|
213
203
|
it 'should place files in the root without a bunk empty folder name' do
|
214
204
|
s3.directories[0].files.head(file_name).should_not be_nil
|
215
205
|
end
|
216
206
|
end
|
217
207
|
|
218
208
|
context 'when remote dir starts with a /' do
|
219
|
-
let(:remote_dir) {'/starts_with_slash'}
|
209
|
+
let(:remote_dir) { '/starts_with_slash' }
|
220
210
|
it 'should place files in the root without a bunk empty folder name' do
|
221
211
|
s3.directories[0].files.head('starts_with_slash/test.out').should_not be_nil
|
222
212
|
end
|
223
213
|
end
|
224
214
|
end
|
225
215
|
|
216
|
+
context 'when the file does not exist' do
|
217
|
+
it 'should raise an error' do
|
218
|
+
expect {
|
219
|
+
sync_to_s3.send(:sync_file, 'NO_FILE', '_')
|
220
|
+
}.to raise_error(Elasticity::NoFileError, "File 'NO_FILE' does not exist")
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
context 'when the bucket does not exist' do
|
225
|
+
let(:no_bucket_sync) { Elasticity::SyncToS3.new('NEW_BUCKET', '_', '_') }
|
226
|
+
before do
|
227
|
+
no_bucket_sync.stub(:s3).and_return(s3)
|
228
|
+
FileUtils.mkdir('GOOD_DIR')
|
229
|
+
FileUtils.touch(File.join(%w(GOOD_DIR file_1)))
|
230
|
+
end
|
231
|
+
it 'should create the bucket in s3' do
|
232
|
+
no_bucket_sync.sync('GOOD_DIR', '_')
|
233
|
+
s3.directories.find { |d| d.key == 'NEW_BUCKET' }.should_not be_nil
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
226
237
|
end
|
227
238
|
|
228
239
|
describe '#s3' do
|
229
|
-
let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret') }
|
240
|
+
let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret', 'region') }
|
230
241
|
it 'should connect to S3 using the specified credentials' do
|
231
242
|
Fog::Storage.should_receive(:new).with({
|
232
243
|
:provider => 'AWS',
|
244
|
+
:region => 'region',
|
233
245
|
:aws_access_key_id => 'access',
|
234
246
|
:aws_secret_access_key => 'secret'
|
235
247
|
}).and_return('GOOD_CONNECTION')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '2.
|
4
|
+
version: '2.5'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-09-
|
12
|
+
date: 2012-09-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rest-client
|
@@ -64,17 +64,17 @@ dependencies:
|
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
65
65
|
none: false
|
66
66
|
requirements:
|
67
|
-
- -
|
67
|
+
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
69
|
+
version: '0.9'
|
70
70
|
type: :development
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- -
|
75
|
+
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: '0'
|
77
|
+
version: '0.9'
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: rspec
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,7 +155,6 @@ files:
|
|
155
155
|
- lib/elasticity/job_flow_step.rb
|
156
156
|
- lib/elasticity/pig_step.rb
|
157
157
|
- lib/elasticity/streaming_step.rb
|
158
|
-
- lib/elasticity/support/conditional_raise.rb
|
159
158
|
- lib/elasticity/sync_to_s3.rb
|
160
159
|
- lib/elasticity/version.rb
|
161
160
|
- spec/lib/elasticity/aws_request_spec.rb
|
@@ -173,7 +172,6 @@ files:
|
|
173
172
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
174
173
|
- spec/lib/elasticity/pig_step_spec.rb
|
175
174
|
- spec/lib/elasticity/streaming_step_spec.rb
|
176
|
-
- spec/lib/elasticity/support/conditional_raise_spec.rb
|
177
175
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
178
176
|
- spec/spec_helper.rb
|
179
177
|
- spec/support/be_a_hash_including_matcher.rb
|
@@ -191,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
191
189
|
version: '0'
|
192
190
|
segments:
|
193
191
|
- 0
|
194
|
-
hash: -
|
192
|
+
hash: -4541363325950914992
|
195
193
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
196
194
|
none: false
|
197
195
|
requirements:
|
@@ -200,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
200
198
|
version: '0'
|
201
199
|
segments:
|
202
200
|
- 0
|
203
|
-
hash: -
|
201
|
+
hash: -4541363325950914992
|
204
202
|
requirements: []
|
205
203
|
rubyforge_project:
|
206
204
|
rubygems_version: 1.8.24
|
@@ -223,7 +221,6 @@ test_files:
|
|
223
221
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
224
222
|
- spec/lib/elasticity/pig_step_spec.rb
|
225
223
|
- spec/lib/elasticity/streaming_step_spec.rb
|
226
|
-
- spec/lib/elasticity/support/conditional_raise_spec.rb
|
227
224
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
228
225
|
- spec/spec_helper.rb
|
229
226
|
- spec/support/be_a_hash_including_matcher.rb
|
@@ -1,23 +0,0 @@
|
|
1
|
-
module Elasticity
|
2
|
-
|
3
|
-
module ConditionalRaising
|
4
|
-
|
5
|
-
def raise_if(conditional, error_class, message)
|
6
|
-
raise error_class, message if conditional
|
7
|
-
end
|
8
|
-
|
9
|
-
def raise_unless(conditional, error_class, message)
|
10
|
-
raise error_class, message unless conditional
|
11
|
-
end
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
module Kernel
|
18
|
-
include Elasticity::ConditionalRaising
|
19
|
-
end
|
20
|
-
|
21
|
-
class Object
|
22
|
-
include Kernel
|
23
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
describe 'Conditional Raising' do
|
2
|
-
|
3
|
-
describe '#raise_if' do
|
4
|
-
|
5
|
-
it 'should raise the specified error if the condition is true' do
|
6
|
-
expect {
|
7
|
-
raise_if true, RuntimeError, 'MESSAGE'
|
8
|
-
}.to raise_error(RuntimeError, 'MESSAGE')
|
9
|
-
end
|
10
|
-
|
11
|
-
it 'should not raise the specified error if the condition is false' do
|
12
|
-
expect {
|
13
|
-
raise_if false, RuntimeError, 'MESSAGE'
|
14
|
-
}.to_not raise_error(RuntimeError, 'MESSAGE')
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
describe '#raise_unless' do
|
20
|
-
|
21
|
-
it 'should not raise the specified error unless the condition is true' do
|
22
|
-
expect {
|
23
|
-
raise_unless true, RuntimeError, 'MESSAGE'
|
24
|
-
}.to_not raise_error(RuntimeError, 'MESSAGE')
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'should raise the specified error unless the condition is false' do
|
28
|
-
expect {
|
29
|
-
raise_unless false, RuntimeError, 'MESSAGE'
|
30
|
-
}.to raise_error(RuntimeError, 'MESSAGE')
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|