elasticity 2.4 → 2.5
Sign up to get free protection for your applications and to get access to all the features.
- data/HISTORY.md +5 -0
- data/README.md +15 -8
- data/elasticity.gemspec +1 -1
- data/lib/elasticity.rb +0 -2
- data/lib/elasticity/aws_request.rb +1 -1
- data/lib/elasticity/hive_step.rb +6 -6
- data/lib/elasticity/instance_group.rb +12 -4
- data/lib/elasticity/job_flow.rb +16 -6
- data/lib/elasticity/sync_to_s3.rb +20 -10
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/aws_request_spec.rb +1 -1
- data/spec/lib/elasticity/hive_step_spec.rb +14 -10
- data/spec/lib/elasticity/job_flow_integration_spec.rb +5 -1
- data/spec/lib/elasticity/sync_to_s3_spec.rb +70 -58
- metadata +8 -11
- data/lib/elasticity/support/conditional_raise.rb +0 -23
- data/spec/lib/elasticity/support/conditional_raise_spec.rb +0 -35
data/HISTORY.md
CHANGED
data/README.md
CHANGED
@@ -16,7 +16,7 @@ gem install elasticity
|
|
16
16
|
or in your Gemfile
|
17
17
|
|
18
18
|
```
|
19
|
-
gem 'elasticity', '~> 2.
|
19
|
+
gem 'elasticity', '~> 2.5'
|
20
20
|
```
|
21
21
|
|
22
22
|
This will ensure that you protect yourself from API changes, which will only be made in major revisions.
|
@@ -273,20 +273,27 @@ jobflow.add_step(jar_step)
|
|
273
273
|
|
274
274
|
## 6 - Upload Assets (optional)
|
275
275
|
|
276
|
-
This isn't part of ```JobFlow```; more of an aside
|
276
|
+
This isn't part of ```JobFlow```; more of an aside. Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. Most commonly this will be a set of resources to run the job (e.g. JAR files, streaming scripts, etc.) and a set of resources used by the job itself (e.g. a TSV file with a range of valid values, join tables, etc.).
|
277
277
|
|
278
278
|
```ruby
|
279
|
-
# Specify the bucket
|
280
|
-
s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret')
|
279
|
+
# Specify the bucket name, AWS credentials and region
|
280
|
+
s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret', 'region')
|
281
281
|
|
282
|
-
#
|
282
|
+
# Alternatively, specify nothing :)
|
283
|
+
# - Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
|
284
|
+
# - Use the 'us-east-1' region by default
|
283
285
|
# s3 = Elasticity::SyncToS3('my-bucket')
|
284
286
|
|
285
|
-
# Recursively sync the contents of '/
|
286
|
-
s3.sync('/
|
287
|
+
# Recursively sync the contents of '/foo' under the remote location 'remote-dir/this-job'
|
288
|
+
s3.sync('/foo', 'remote-dir/this-job')
|
289
|
+
|
290
|
+
# Sync a single file to a remote directory
|
291
|
+
s3.sync('/foo/this-job/tables/join.tsv', 'remote-dir/this-job/tables')
|
287
292
|
```
|
288
293
|
|
289
|
-
If the
|
294
|
+
If the bucket doesn't exist, it will be created.
|
295
|
+
|
296
|
+
If a file already exists, there is an MD5 checksum evaluation. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/tables/join.tsv``` in your EMR jobs.
|
290
297
|
|
291
298
|
## 7 - Run the Job Flow
|
292
299
|
|
data/elasticity.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.add_dependency('nokogiri')
|
16
16
|
s.add_dependency('fog')
|
17
17
|
|
18
|
-
s.add_development_dependency('rake')
|
18
|
+
s.add_development_dependency('rake', '~> 0.9')
|
19
19
|
s.add_development_dependency('rspec', '~> 2.11.0')
|
20
20
|
s.add_development_dependency('timecop')
|
21
21
|
s.add_development_dependency('fakefs', '~> 0.4')
|
data/lib/elasticity.rb
CHANGED
@@ -50,7 +50,7 @@ module Elasticity
|
|
50
50
|
def get_secret_key(secret)
|
51
51
|
return secret if secret
|
52
52
|
return ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
|
53
|
-
raise MissingKeyError, 'Please provide a secret key or set
|
53
|
+
raise MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.'
|
54
54
|
end
|
55
55
|
|
56
56
|
# (Used from RightScale's right_aws gem.)
|
data/lib/elasticity/hive_step.rb
CHANGED
@@ -17,7 +17,7 @@ module Elasticity
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def to_aws_step(job_flow)
|
20
|
-
args = %w(s3://elasticmapreduce/libs/hive/hive-script --run-hive-script --args)
|
20
|
+
args = %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions latest --run-hive-script --args)
|
21
21
|
args.concat(['-f', @script])
|
22
22
|
@variables.keys.sort.each do |name|
|
23
23
|
args.concat(['-d', "#{name}=#{@variables[name]}"])
|
@@ -43,11 +43,11 @@ module Elasticity
|
|
43
43
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
44
44
|
:args => [
|
45
45
|
's3://elasticmapreduce/libs/hive/hive-script',
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
46
|
+
'--base-path',
|
47
|
+
's3://elasticmapreduce/libs/hive/',
|
48
|
+
'--install-hive',
|
49
|
+
'--hive-versions',
|
50
|
+
'latest'
|
51
51
|
],
|
52
52
|
},
|
53
53
|
:name => 'Elasticity - Install Hive'
|
@@ -19,19 +19,27 @@ module Elasticity
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def count=(instance_count)
|
22
|
-
|
23
|
-
|
22
|
+
if instance_count <= 0
|
23
|
+
raise ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
|
24
|
+
end
|
25
|
+
if @role == 'MASTER' && instance_count != 1
|
26
|
+
raise ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
|
27
|
+
end
|
24
28
|
@count = instance_count
|
25
29
|
end
|
26
30
|
|
27
31
|
def role=(group_role)
|
28
|
-
|
32
|
+
if !ROLES.include?(group_role)
|
33
|
+
raise ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
|
34
|
+
end
|
29
35
|
@count = 1 if group_role == 'MASTER'
|
30
36
|
@role = group_role
|
31
37
|
end
|
32
38
|
|
33
39
|
def set_spot_instances(bid_price)
|
34
|
-
|
40
|
+
if bid_price < 0
|
41
|
+
raise ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
|
42
|
+
end
|
35
43
|
@bid_price = bid_price
|
36
44
|
@market = 'SPOT'
|
37
45
|
end
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -21,7 +21,7 @@ module Elasticity
|
|
21
21
|
|
22
22
|
attr_reader :access_key
|
23
23
|
attr_reader :secret_key
|
24
|
-
|
24
|
+
|
25
25
|
def initialize(access=nil, secret=nil)
|
26
26
|
@action_on_failure = 'TERMINATE_JOB_FLOW'
|
27
27
|
@hadoop_version = '1.0.3'
|
@@ -73,7 +73,9 @@ module Elasticity
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def add_bootstrap_action(bootstrap_action)
|
76
|
-
|
76
|
+
if is_jobflow_running?
|
77
|
+
raise JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
|
78
|
+
end
|
77
79
|
@bootstrap_actions << bootstrap_action
|
78
80
|
end
|
79
81
|
|
@@ -106,18 +108,26 @@ module Elasticity
|
|
106
108
|
end
|
107
109
|
|
108
110
|
def run
|
109
|
-
|
110
|
-
|
111
|
+
if @jobflow_steps.empty?
|
112
|
+
raise JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
|
113
|
+
end
|
114
|
+
if is_jobflow_running?
|
115
|
+
raise JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
|
116
|
+
end
|
111
117
|
@jobflow_id = emr.run_job_flow(jobflow_config)
|
112
118
|
end
|
113
119
|
|
114
120
|
def shutdown
|
115
|
-
|
121
|
+
if !is_jobflow_running?
|
122
|
+
raise JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
|
123
|
+
end
|
116
124
|
emr.terminate_jobflows(@jobflow_id)
|
117
125
|
end
|
118
126
|
|
119
127
|
def status
|
120
|
-
|
128
|
+
if !is_jobflow_running?
|
129
|
+
raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
|
130
|
+
end
|
121
131
|
emr.describe_jobflow(@jobflow_id)
|
122
132
|
end
|
123
133
|
|
@@ -2,38 +2,45 @@ module Elasticity
|
|
2
2
|
|
3
3
|
class NoBucketError < StandardError; end
|
4
4
|
class NoDirectoryError < StandardError; end
|
5
|
+
class NoFileError < StandardError; end
|
5
6
|
|
6
7
|
class SyncToS3
|
7
8
|
|
8
9
|
attr_reader :access_key
|
9
10
|
attr_reader :secret_key
|
10
11
|
attr_reader :bucket_name
|
12
|
+
attr_reader :region
|
11
13
|
|
12
|
-
def initialize(bucket, access=nil, secret=nil)
|
14
|
+
def initialize(bucket, access=nil, secret=nil, region=nil)
|
13
15
|
@access_key = get_access_key(access)
|
14
16
|
@secret_key = get_secret_key(secret)
|
15
17
|
@bucket_name = bucket
|
18
|
+
@region = region ||= 'us-east-1'
|
16
19
|
end
|
17
20
|
|
18
21
|
def sync(local, remote)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
+
if File.directory?(local)
|
23
|
+
sync_dir(local, remote)
|
24
|
+
else
|
25
|
+
sync_file(local, remote)
|
26
|
+
end
|
22
27
|
end
|
23
28
|
|
24
29
|
private
|
25
30
|
|
26
|
-
def sync_dir(
|
27
|
-
|
31
|
+
def sync_dir(dir_name, remote_dir)
|
32
|
+
raise NoDirectoryError, "Directory '#{dir_name}' does not exist or is not a directory" unless File.directory?(dir_name)
|
33
|
+
Dir.glob(File.join([dir_name, '*'])).each do |entry|
|
28
34
|
if File.directory?(entry)
|
29
|
-
sync_dir(entry, [
|
35
|
+
sync_dir(entry, [remote_dir, File.basename(entry)].join('/'))
|
30
36
|
else
|
31
|
-
sync_file(entry,
|
37
|
+
sync_file(entry, remote_dir)
|
32
38
|
end
|
33
39
|
end
|
34
40
|
end
|
35
41
|
|
36
42
|
def sync_file(file_name, remote_dir)
|
43
|
+
raise NoFileError, "File '#{file_name}' does not exist" unless File.exists?(file_name)
|
37
44
|
remote_dir = remote_dir.gsub(/^(\/)/, '')
|
38
45
|
remote_path = (remote_dir.empty?) ? (File.basename(file_name)) : [remote_dir, File.basename(file_name)].join('/')
|
39
46
|
metadata = bucket.files.head(remote_path)
|
@@ -46,14 +53,17 @@ module Elasticity
|
|
46
53
|
})
|
47
54
|
end
|
48
55
|
|
56
|
+
private
|
57
|
+
|
49
58
|
def bucket
|
50
|
-
|
51
|
-
@bucket ||=
|
59
|
+
@bucket ||= s3.directories.find { |d| d.key == @bucket_name }
|
60
|
+
@bucket ||= s3.directories.create(:key => @bucket_name)
|
52
61
|
end
|
53
62
|
|
54
63
|
def s3
|
55
64
|
@connection ||= Fog::Storage.new({
|
56
65
|
:provider => 'AWS',
|
66
|
+
:region => @region,
|
57
67
|
:aws_access_key_id => @access_key,
|
58
68
|
:aws_secret_access_key => @secret_key
|
59
69
|
})
|
data/lib/elasticity/version.rb
CHANGED
@@ -68,7 +68,7 @@ describe Elasticity::AwsRequest do
|
|
68
68
|
it 'should raise an error' do
|
69
69
|
expect {
|
70
70
|
missing_something.access_key
|
71
|
-
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set
|
71
|
+
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
@@ -20,10 +20,14 @@ describe Elasticity::HiveStep do
|
|
20
20
|
step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
|
21
21
|
step[:hadoop_jar_step][:args].should start_with([
|
22
22
|
's3://elasticmapreduce/libs/hive/hive-script',
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
23
|
+
'--base-path',
|
24
|
+
's3://elasticmapreduce/libs/hive/',
|
25
|
+
'--hive-versions',
|
26
|
+
'latest',
|
27
|
+
'--run-hive-script',
|
28
|
+
'--args',
|
29
|
+
'-f',
|
30
|
+
'script.hql'
|
27
31
|
])
|
28
32
|
end
|
29
33
|
|
@@ -39,7 +43,7 @@ describe Elasticity::HiveStep do
|
|
39
43
|
|
40
44
|
it 'should convert to aws step format' do
|
41
45
|
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
|
42
|
-
step[:hadoop_jar_step][:args][
|
46
|
+
step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
|
43
47
|
end
|
44
48
|
end
|
45
49
|
|
@@ -60,11 +64,11 @@ describe Elasticity::HiveStep do
|
|
60
64
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
61
65
|
:args => [
|
62
66
|
's3://elasticmapreduce/libs/hive/hive-script',
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
67
|
+
'--base-path',
|
68
|
+
's3://elasticmapreduce/libs/hive/',
|
69
|
+
'--install-hive',
|
70
|
+
'--hive-versions',
|
71
|
+
'latest'
|
68
72
|
],
|
69
73
|
},
|
70
74
|
:name => 'Elasticity - Install Hive'
|
@@ -68,7 +68,11 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
68
68
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
69
69
|
:args => [
|
70
70
|
's3://elasticmapreduce/libs/hive/hive-script',
|
71
|
-
|
71
|
+
'--base-path',
|
72
|
+
's3://elasticmapreduce/libs/hive/',
|
73
|
+
'--hive-versions',
|
74
|
+
'latest',
|
75
|
+
'--run-hive-script',
|
72
76
|
'--args',
|
73
77
|
'-f', 's3n://slif-hive/test.q',
|
74
78
|
'-d', 'OUTPUT=s3n://slif-test/output'
|
@@ -17,18 +17,19 @@ describe Elasticity::SyncToS3 do
|
|
17
17
|
describe 'basic assignment' do
|
18
18
|
|
19
19
|
it 'should set the proper values' do
|
20
|
-
sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret')
|
20
|
+
sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret', 'region')
|
21
21
|
sync.access_key.should == 'access'
|
22
22
|
sync.secret_key.should == 'secret'
|
23
23
|
sync.bucket_name.should == 'bucket'
|
24
|
+
sync.region.should == 'region'
|
24
25
|
end
|
25
26
|
|
26
27
|
end
|
27
28
|
|
28
|
-
context 'when
|
29
|
+
context 'when parameters are missing' do
|
29
30
|
|
30
|
-
let(:
|
31
|
-
let(:
|
31
|
+
let(:optional_params_nil) { Elasticity::SyncToS3.new('_', nil, nil, nil) }
|
32
|
+
let(:optional_params_missing) { Elasticity::SyncToS3.new('_') }
|
32
33
|
|
33
34
|
before do
|
34
35
|
ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return(access_key)
|
@@ -38,13 +39,21 @@ describe Elasticity::SyncToS3 do
|
|
38
39
|
context 'when environment variables are present' do
|
39
40
|
let(:access_key) { 'ENV_ACCESS' }
|
40
41
|
let(:secret_key) { 'ENV_SECRET' }
|
41
|
-
it 'should assign them to the keys' do
|
42
|
-
both_keys_nil.access_key.should == 'ENV_ACCESS'
|
43
|
-
both_keys_nil.secret_key.should == 'ENV_SECRET'
|
44
42
|
|
45
|
-
|
46
|
-
|
43
|
+
it 'should assign environment variables to the keys' do
|
44
|
+
optional_params_nil.access_key.should == 'ENV_ACCESS'
|
45
|
+
optional_params_nil.secret_key.should == 'ENV_SECRET'
|
46
|
+
optional_params_missing.access_key.should == 'ENV_ACCESS'
|
47
|
+
optional_params_missing.secret_key.should == 'ENV_SECRET'
|
47
48
|
end
|
49
|
+
|
50
|
+
it 'should set the region to us-east-1' do
|
51
|
+
optional_params_nil.region.should == 'us-east-1'
|
52
|
+
optional_params_nil.region.should == 'us-east-1'
|
53
|
+
optional_params_missing.region.should == 'us-east-1'
|
54
|
+
optional_params_missing.region.should == 'us-east-1'
|
55
|
+
end
|
56
|
+
|
48
57
|
end
|
49
58
|
|
50
59
|
context 'when environment variables are not present' do
|
@@ -54,7 +63,7 @@ describe Elasticity::SyncToS3 do
|
|
54
63
|
let(:secret_key) { '_' }
|
55
64
|
it 'should raise an error' do
|
56
65
|
expect {
|
57
|
-
|
66
|
+
optional_params_nil # Trigger instantiation
|
58
67
|
}.to raise_error(Elasticity::MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.')
|
59
68
|
end
|
60
69
|
end
|
@@ -64,7 +73,7 @@ describe Elasticity::SyncToS3 do
|
|
64
73
|
let(:secret_key) { nil }
|
65
74
|
it 'should raise an error' do
|
66
75
|
expect {
|
67
|
-
|
76
|
+
optional_params_nil # Trigger instantiation
|
68
77
|
}.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
|
69
78
|
end
|
70
79
|
end
|
@@ -77,50 +86,23 @@ describe Elasticity::SyncToS3 do
|
|
77
86
|
|
78
87
|
describe '#sync' do
|
79
88
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
s3.directories.create(:key => bucket_name)
|
84
|
-
end
|
85
|
-
|
86
|
-
context 'when the local directory exists' do
|
87
|
-
before do
|
88
|
-
FileUtils.mkdir('GOOD_DIR')
|
89
|
-
end
|
90
|
-
it 'should sync that directory' do
|
91
|
-
sync_to_s3.should_receive(:sync_dir).with('GOOD_DIR', 'REMOTE_DIR')
|
92
|
-
sync_to_s3.sync('GOOD_DIR', 'REMOTE_DIR')
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
context 'when the local directory does not exist' do
|
97
|
-
it 'should raise an error' do
|
98
|
-
expect {
|
99
|
-
sync_to_s3.sync('BAD_DIR', '_')
|
100
|
-
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'BAD_DIR' does not exist or is not a directory")
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
context 'when the local directory is not a directory' do
|
105
|
-
before do
|
106
|
-
FileUtils.touch('NOT_A_DIR')
|
107
|
-
end
|
108
|
-
it 'should raise an error' do
|
109
|
-
expect {
|
110
|
-
sync_to_s3.sync('NOT_A_DIR', '_')
|
111
|
-
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
|
112
|
-
end
|
113
|
-
end
|
89
|
+
let(:dir_name) { 'GOOD_DIR' }
|
90
|
+
let(:file_name) { File.join(%w(GOOD_DIR file_1)) }
|
91
|
+
let(:remote_dir) { 'REMOTE_DIR' }
|
114
92
|
|
93
|
+
before do
|
94
|
+
FileUtils.mkdir(dir_name)
|
95
|
+
FileUtils.touch(file_name)
|
96
|
+
end
|
97
|
+
|
98
|
+
it 'should sync directories' do
|
99
|
+
sync_to_s3.should_receive(:sync_dir).with(dir_name, remote_dir)
|
100
|
+
sync_to_s3.sync(dir_name, remote_dir)
|
115
101
|
end
|
116
102
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
expect {
|
121
|
-
sync_to_s3.sync('_', '_')
|
122
|
-
}.to raise_error(Elasticity::NoBucketError, "Bucket 'BAD_BUCKET' does not exist")
|
123
|
-
end
|
103
|
+
it 'should sync files' do
|
104
|
+
sync_to_s3.should_receive(:sync_file).with(file_name, remote_dir)
|
105
|
+
sync_to_s3.sync(file_name, remote_dir)
|
124
106
|
end
|
125
107
|
|
126
108
|
end
|
@@ -156,6 +138,14 @@ describe Elasticity::SyncToS3 do
|
|
156
138
|
end
|
157
139
|
end
|
158
140
|
|
141
|
+
context 'when the directory does not exist or is not a directory' do
|
142
|
+
it 'should raise an error' do
|
143
|
+
expect {
|
144
|
+
sync_to_s3.send(:sync_dir, 'NOT_A_DIR', '_')
|
145
|
+
}.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
159
149
|
end
|
160
150
|
|
161
151
|
describe '#sync_file' do
|
@@ -164,13 +154,13 @@ describe Elasticity::SyncToS3 do
|
|
164
154
|
let(:file_name) { 'test.out' }
|
165
155
|
let(:full_path) { File.join([local_dir, file_name]) }
|
166
156
|
let(:remote_dir) { 'job/assets' }
|
167
|
-
let(:remote_path) { "#{remote_dir}/#{file_name}"}
|
157
|
+
let(:remote_path) { "#{remote_dir}/#{file_name}" }
|
168
158
|
let(:file_data) { 'Some test content' }
|
169
159
|
|
170
160
|
before do
|
171
161
|
s3.directories.create(:key => bucket_name)
|
172
162
|
FileUtils.makedirs(local_dir)
|
173
|
-
File.open(full_path, 'w') {|f| f.write(file_data) }
|
163
|
+
File.open(full_path, 'w') { |f| f.write(file_data) }
|
174
164
|
end
|
175
165
|
|
176
166
|
it 'should write the specified file into the remote directory' do
|
@@ -202,34 +192,56 @@ describe Elasticity::SyncToS3 do
|
|
202
192
|
end
|
203
193
|
|
204
194
|
context 'when remote dir is empty' do
|
205
|
-
let(:remote_dir) {''}
|
195
|
+
let(:remote_dir) { '' }
|
206
196
|
it 'should place files in the root without a bunk empty folder name' do
|
207
197
|
s3.directories[0].files.head(file_name).should_not be_nil
|
208
198
|
end
|
209
199
|
end
|
210
200
|
|
211
201
|
context 'when remote dir is /' do
|
212
|
-
let(:remote_dir) {'/'}
|
202
|
+
let(:remote_dir) { '/' }
|
213
203
|
it 'should place files in the root without a bunk empty folder name' do
|
214
204
|
s3.directories[0].files.head(file_name).should_not be_nil
|
215
205
|
end
|
216
206
|
end
|
217
207
|
|
218
208
|
context 'when remote dir starts with a /' do
|
219
|
-
let(:remote_dir) {'/starts_with_slash'}
|
209
|
+
let(:remote_dir) { '/starts_with_slash' }
|
220
210
|
it 'should place files in the root without a bunk empty folder name' do
|
221
211
|
s3.directories[0].files.head('starts_with_slash/test.out').should_not be_nil
|
222
212
|
end
|
223
213
|
end
|
224
214
|
end
|
225
215
|
|
216
|
+
context 'when the file does not exist' do
|
217
|
+
it 'should raise an error' do
|
218
|
+
expect {
|
219
|
+
sync_to_s3.send(:sync_file, 'NO_FILE', '_')
|
220
|
+
}.to raise_error(Elasticity::NoFileError, "File 'NO_FILE' does not exist")
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
context 'when the bucket does not exist' do
|
225
|
+
let(:no_bucket_sync) { Elasticity::SyncToS3.new('NEW_BUCKET', '_', '_') }
|
226
|
+
before do
|
227
|
+
no_bucket_sync.stub(:s3).and_return(s3)
|
228
|
+
FileUtils.mkdir('GOOD_DIR')
|
229
|
+
FileUtils.touch(File.join(%w(GOOD_DIR file_1)))
|
230
|
+
end
|
231
|
+
it 'should create the bucket in s3' do
|
232
|
+
no_bucket_sync.sync('GOOD_DIR', '_')
|
233
|
+
s3.directories.find { |d| d.key == 'NEW_BUCKET' }.should_not be_nil
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
226
237
|
end
|
227
238
|
|
228
239
|
describe '#s3' do
|
229
|
-
let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret') }
|
240
|
+
let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret', 'region') }
|
230
241
|
it 'should connect to S3 using the specified credentials' do
|
231
242
|
Fog::Storage.should_receive(:new).with({
|
232
243
|
:provider => 'AWS',
|
244
|
+
:region => 'region',
|
233
245
|
:aws_access_key_id => 'access',
|
234
246
|
:aws_secret_access_key => 'secret'
|
235
247
|
}).and_return('GOOD_CONNECTION')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '2.
|
4
|
+
version: '2.5'
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-09-
|
12
|
+
date: 2012-09-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rest-client
|
@@ -64,17 +64,17 @@ dependencies:
|
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
65
65
|
none: false
|
66
66
|
requirements:
|
67
|
-
- -
|
67
|
+
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
69
|
+
version: '0.9'
|
70
70
|
type: :development
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
73
73
|
none: false
|
74
74
|
requirements:
|
75
|
-
- -
|
75
|
+
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: '0'
|
77
|
+
version: '0.9'
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: rspec
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,7 +155,6 @@ files:
|
|
155
155
|
- lib/elasticity/job_flow_step.rb
|
156
156
|
- lib/elasticity/pig_step.rb
|
157
157
|
- lib/elasticity/streaming_step.rb
|
158
|
-
- lib/elasticity/support/conditional_raise.rb
|
159
158
|
- lib/elasticity/sync_to_s3.rb
|
160
159
|
- lib/elasticity/version.rb
|
161
160
|
- spec/lib/elasticity/aws_request_spec.rb
|
@@ -173,7 +172,6 @@ files:
|
|
173
172
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
174
173
|
- spec/lib/elasticity/pig_step_spec.rb
|
175
174
|
- spec/lib/elasticity/streaming_step_spec.rb
|
176
|
-
- spec/lib/elasticity/support/conditional_raise_spec.rb
|
177
175
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
178
176
|
- spec/spec_helper.rb
|
179
177
|
- spec/support/be_a_hash_including_matcher.rb
|
@@ -191,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
191
189
|
version: '0'
|
192
190
|
segments:
|
193
191
|
- 0
|
194
|
-
hash: -
|
192
|
+
hash: -4541363325950914992
|
195
193
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
196
194
|
none: false
|
197
195
|
requirements:
|
@@ -200,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
200
198
|
version: '0'
|
201
199
|
segments:
|
202
200
|
- 0
|
203
|
-
hash: -
|
201
|
+
hash: -4541363325950914992
|
204
202
|
requirements: []
|
205
203
|
rubyforge_project:
|
206
204
|
rubygems_version: 1.8.24
|
@@ -223,7 +221,6 @@ test_files:
|
|
223
221
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
224
222
|
- spec/lib/elasticity/pig_step_spec.rb
|
225
223
|
- spec/lib/elasticity/streaming_step_spec.rb
|
226
|
-
- spec/lib/elasticity/support/conditional_raise_spec.rb
|
227
224
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
228
225
|
- spec/spec_helper.rb
|
229
226
|
- spec/support/be_a_hash_including_matcher.rb
|
@@ -1,23 +0,0 @@
|
|
1
|
-
module Elasticity
|
2
|
-
|
3
|
-
module ConditionalRaising
|
4
|
-
|
5
|
-
def raise_if(conditional, error_class, message)
|
6
|
-
raise error_class, message if conditional
|
7
|
-
end
|
8
|
-
|
9
|
-
def raise_unless(conditional, error_class, message)
|
10
|
-
raise error_class, message unless conditional
|
11
|
-
end
|
12
|
-
|
13
|
-
end
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
module Kernel
|
18
|
-
include Elasticity::ConditionalRaising
|
19
|
-
end
|
20
|
-
|
21
|
-
class Object
|
22
|
-
include Kernel
|
23
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
describe 'Conditional Raising' do
|
2
|
-
|
3
|
-
describe '#raise_if' do
|
4
|
-
|
5
|
-
it 'should raise the specified error if the condition is true' do
|
6
|
-
expect {
|
7
|
-
raise_if true, RuntimeError, 'MESSAGE'
|
8
|
-
}.to raise_error(RuntimeError, 'MESSAGE')
|
9
|
-
end
|
10
|
-
|
11
|
-
it 'should not raise the specified error if the condition is false' do
|
12
|
-
expect {
|
13
|
-
raise_if false, RuntimeError, 'MESSAGE'
|
14
|
-
}.to_not raise_error(RuntimeError, 'MESSAGE')
|
15
|
-
end
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
describe '#raise_unless' do
|
20
|
-
|
21
|
-
it 'should not raise the specified error unless the condition is true' do
|
22
|
-
expect {
|
23
|
-
raise_unless true, RuntimeError, 'MESSAGE'
|
24
|
-
}.to_not raise_error(RuntimeError, 'MESSAGE')
|
25
|
-
end
|
26
|
-
|
27
|
-
it 'should raise the specified error unless the condition is false' do
|
28
|
-
expect {
|
29
|
-
raise_unless false, RuntimeError, 'MESSAGE'
|
30
|
-
}.to raise_error(RuntimeError, 'MESSAGE')
|
31
|
-
end
|
32
|
-
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|