elasticity 2.4 → 2.5

Sign up to get free protection for your applications and to get access to all the features.
data/HISTORY.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 2.5 - September 29, 2012
2
+
3
+ + ```SyncToS3``` supports S3 region specification.
4
+ + ```SyncToS3#sync``` supports being called with both files and directories.
5
+
1
6
  ## 2.4 - September 1, 2012
2
7
 
3
8
  + ```SyncToS3``` added to enable one-way asset synchronization.
data/README.md CHANGED
@@ -16,7 +16,7 @@ gem install elasticity
16
16
  or in your Gemfile
17
17
 
18
18
  ```
19
- gem 'elasticity', '~> 2.0'
19
+ gem 'elasticity', '~> 2.5'
20
20
  ```
21
21
 
22
22
  This will ensure that you protect yourself from API changes, which will only be made in major revisions.
@@ -273,20 +273,27 @@ jobflow.add_step(jar_step)
273
273
 
274
274
  ## 6 - Upload Assets (optional)
275
275
 
276
- This isn't part of ```JobFlow```; more of an aside :) Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. For example, a TSV file with a range of valid values, join tables, etc.
276
+ This isn't part of ```JobFlow```; more of an aside. Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. Most commonly this will be a set of resources to run the job (e.g. JAR files, streaming scripts, etc.) and a set of resources used by the job itself (e.g. a TSV file with a range of valid values, join tables, etc.).
277
277
 
278
278
  ```ruby
279
- # Specify the bucket and AWS credentials
280
- s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret')
279
+ # Specify the bucket name, AWS credentials and region
280
+ s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret', 'region')
281
281
 
282
- # Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
282
+ # Alternatively, specify nothing :)
283
+ # - Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
284
+ # - Use the 'us-east-1' region by default
283
285
  # s3 = Elasticity::SyncToS3('my-bucket')
284
286
 
285
- # Recursively sync the contents of '/some/parent/dir' under the remote location 'remote-dir/this-job/assets'
286
- s3.sync('/some/parent/dir', 'remote-dir/this-job/assets')
287
+ # Recursively sync the contents of '/foo' under the remote location 'remote-dir/this-job'
288
+ s3.sync('/foo', 'remote-dir/this-job')
289
+
290
+ # Sync a single file to a remote directory
291
+ s3.sync('/foo/this-job/tables/join.tsv', 'remote-dir/this-job/tables')
287
292
  ```
288
293
 
289
- If the files already exist, there is an MD5 checksum check. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/assets/join.tsv``` in your EMR jobs.
294
+ If the bucket doesn't exist, it will be created.
295
+
296
+ If a file already exists, there is an MD5 checksum evaluation. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/tables/join.tsv``` in your EMR jobs.
290
297
 
291
298
  ## 7 - Run the Job Flow
292
299
 
data/elasticity.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
15
15
  s.add_dependency('nokogiri')
16
16
  s.add_dependency('fog')
17
17
 
18
- s.add_development_dependency('rake')
18
+ s.add_development_dependency('rake', '~> 0.9')
19
19
  s.add_development_dependency('rspec', '~> 2.11.0')
20
20
  s.add_development_dependency('timecop')
21
21
  s.add_development_dependency('fakefs', '~> 0.4')
data/lib/elasticity.rb CHANGED
@@ -5,8 +5,6 @@ require 'rest_client'
5
5
  require 'nokogiri'
6
6
  require 'fog'
7
7
 
8
- require 'elasticity/support/conditional_raise'
9
-
10
8
  require 'elasticity/aws_request'
11
9
  require 'elasticity/emr'
12
10
 
@@ -50,7 +50,7 @@ module Elasticity
50
50
  def get_secret_key(secret)
51
51
  return secret if secret
52
52
  return ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
53
- raise MissingKeyError, 'Please provide a secret key or set AWS_ACCESS_KEY_ID.'
53
+ raise MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.'
54
54
  end
55
55
 
56
56
  # (Used from RightScale's right_aws gem.)
@@ -17,7 +17,7 @@ module Elasticity
17
17
  end
18
18
 
19
19
  def to_aws_step(job_flow)
20
- args = %w(s3://elasticmapreduce/libs/hive/hive-script --run-hive-script --args)
20
+ args = %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions latest --run-hive-script --args)
21
21
  args.concat(['-f', @script])
22
22
  @variables.keys.sort.each do |name|
23
23
  args.concat(['-d', "#{name}=#{@variables[name]}"])
@@ -43,11 +43,11 @@ module Elasticity
43
43
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
44
44
  :args => [
45
45
  's3://elasticmapreduce/libs/hive/hive-script',
46
- '--base-path',
47
- 's3://elasticmapreduce/libs/hive/',
48
- '--install-hive',
49
- '--hive-versions',
50
- 'latest'
46
+ '--base-path',
47
+ 's3://elasticmapreduce/libs/hive/',
48
+ '--install-hive',
49
+ '--hive-versions',
50
+ 'latest'
51
51
  ],
52
52
  },
53
53
  :name => 'Elasticity - Install Hive'
@@ -19,19 +19,27 @@ module Elasticity
19
19
  end
20
20
 
21
21
  def count=(instance_count)
22
- raise_if instance_count <= 0, ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
23
- raise_if @role == 'MASTER' && instance_count != 1, ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
22
+ if instance_count <= 0
23
+ raise ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
24
+ end
25
+ if @role == 'MASTER' && instance_count != 1
26
+ raise ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
27
+ end
24
28
  @count = instance_count
25
29
  end
26
30
 
27
31
  def role=(group_role)
28
- raise_unless ROLES.include?(group_role), ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
32
+ if !ROLES.include?(group_role)
33
+ raise ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
34
+ end
29
35
  @count = 1 if group_role == 'MASTER'
30
36
  @role = group_role
31
37
  end
32
38
 
33
39
  def set_spot_instances(bid_price)
34
- raise_unless bid_price > 0, ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
40
+ if bid_price < 0
41
+ raise ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
42
+ end
35
43
  @bid_price = bid_price
36
44
  @market = 'SPOT'
37
45
  end
@@ -21,7 +21,7 @@ module Elasticity
21
21
 
22
22
  attr_reader :access_key
23
23
  attr_reader :secret_key
24
-
24
+
25
25
  def initialize(access=nil, secret=nil)
26
26
  @action_on_failure = 'TERMINATE_JOB_FLOW'
27
27
  @hadoop_version = '1.0.3'
@@ -73,7 +73,9 @@ module Elasticity
73
73
  end
74
74
 
75
75
  def add_bootstrap_action(bootstrap_action)
76
- raise_if is_jobflow_running?, JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
76
+ if is_jobflow_running?
77
+ raise JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
78
+ end
77
79
  @bootstrap_actions << bootstrap_action
78
80
  end
79
81
 
@@ -106,18 +108,26 @@ module Elasticity
106
108
  end
107
109
 
108
110
  def run
109
- raise_if @jobflow_steps.empty?, JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
110
- raise_if is_jobflow_running?, JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
111
+ if @jobflow_steps.empty?
112
+ raise JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
113
+ end
114
+ if is_jobflow_running?
115
+ raise JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
116
+ end
111
117
  @jobflow_id = emr.run_job_flow(jobflow_config)
112
118
  end
113
119
 
114
120
  def shutdown
115
- raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
121
+ if !is_jobflow_running?
122
+ raise JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
123
+ end
116
124
  emr.terminate_jobflows(@jobflow_id)
117
125
  end
118
126
 
119
127
  def status
120
- raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
128
+ if !is_jobflow_running?
129
+ raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
130
+ end
121
131
  emr.describe_jobflow(@jobflow_id)
122
132
  end
123
133
 
@@ -2,38 +2,45 @@ module Elasticity
2
2
 
3
3
  class NoBucketError < StandardError; end
4
4
  class NoDirectoryError < StandardError; end
5
+ class NoFileError < StandardError; end
5
6
 
6
7
  class SyncToS3
7
8
 
8
9
  attr_reader :access_key
9
10
  attr_reader :secret_key
10
11
  attr_reader :bucket_name
12
+ attr_reader :region
11
13
 
12
- def initialize(bucket, access=nil, secret=nil)
14
+ def initialize(bucket, access=nil, secret=nil, region=nil)
13
15
  @access_key = get_access_key(access)
14
16
  @secret_key = get_secret_key(secret)
15
17
  @bucket_name = bucket
18
+ @region = region ||= 'us-east-1'
16
19
  end
17
20
 
18
21
  def sync(local, remote)
19
- raise_unless bucket, NoBucketError, "Bucket '#@bucket_name' does not exist"
20
- raise_unless File.directory?(local), NoDirectoryError, "Directory '#{local}' does not exist or is not a directory"
21
- sync_dir(local, remote)
22
+ if File.directory?(local)
23
+ sync_dir(local, remote)
24
+ else
25
+ sync_file(local, remote)
26
+ end
22
27
  end
23
28
 
24
29
  private
25
30
 
26
- def sync_dir(local, remote)
27
- Dir.glob(File.join([local, '*'])).each do |entry|
31
+ def sync_dir(dir_name, remote_dir)
32
+ raise NoDirectoryError, "Directory '#{dir_name}' does not exist or is not a directory" unless File.directory?(dir_name)
33
+ Dir.glob(File.join([dir_name, '*'])).each do |entry|
28
34
  if File.directory?(entry)
29
- sync_dir(entry, [remote, File.basename(entry)].join('/'))
35
+ sync_dir(entry, [remote_dir, File.basename(entry)].join('/'))
30
36
  else
31
- sync_file(entry, remote)
37
+ sync_file(entry, remote_dir)
32
38
  end
33
39
  end
34
40
  end
35
41
 
36
42
  def sync_file(file_name, remote_dir)
43
+ raise NoFileError, "File '#{file_name}' does not exist" unless File.exists?(file_name)
37
44
  remote_dir = remote_dir.gsub(/^(\/)/, '')
38
45
  remote_path = (remote_dir.empty?) ? (File.basename(file_name)) : [remote_dir, File.basename(file_name)].join('/')
39
46
  metadata = bucket.files.head(remote_path)
@@ -46,14 +53,17 @@ module Elasticity
46
53
  })
47
54
  end
48
55
 
56
+ private
57
+
49
58
  def bucket
50
- index = s3.directories.index { |d| d.key == @bucket_name }
51
- @bucket ||= index ? s3.directories[index] : nil
59
+ @bucket ||= s3.directories.find { |d| d.key == @bucket_name }
60
+ @bucket ||= s3.directories.create(:key => @bucket_name)
52
61
  end
53
62
 
54
63
  def s3
55
64
  @connection ||= Fog::Storage.new({
56
65
  :provider => 'AWS',
66
+ :region => @region,
57
67
  :aws_access_key_id => @access_key,
58
68
  :aws_secret_access_key => @secret_key
59
69
  })
@@ -1,3 +1,3 @@
1
1
  module Elasticity
2
- VERSION = '2.4'
2
+ VERSION = '2.5'
3
3
  end
@@ -68,7 +68,7 @@ describe Elasticity::AwsRequest do
68
68
  it 'should raise an error' do
69
69
  expect {
70
70
  missing_something.access_key
71
- }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_ACCESS_KEY_ID.')
71
+ }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
72
72
  end
73
73
  end
74
74
  end
@@ -20,10 +20,14 @@ describe Elasticity::HiveStep do
20
20
  step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
21
21
  step[:hadoop_jar_step][:args].should start_with([
22
22
  's3://elasticmapreduce/libs/hive/hive-script',
23
- '--run-hive-script',
24
- '--args',
25
- '-f',
26
- 'script.hql'
23
+ '--base-path',
24
+ 's3://elasticmapreduce/libs/hive/',
25
+ '--hive-versions',
26
+ 'latest',
27
+ '--run-hive-script',
28
+ '--args',
29
+ '-f',
30
+ 'script.hql'
27
31
  ])
28
32
  end
29
33
 
@@ -39,7 +43,7 @@ describe Elasticity::HiveStep do
39
43
 
40
44
  it 'should convert to aws step format' do
41
45
  step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
42
- step[:hadoop_jar_step][:args][5..9].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
46
+ step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
43
47
  end
44
48
  end
45
49
 
@@ -60,11 +64,11 @@ describe Elasticity::HiveStep do
60
64
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
61
65
  :args => [
62
66
  's3://elasticmapreduce/libs/hive/hive-script',
63
- '--base-path',
64
- 's3://elasticmapreduce/libs/hive/',
65
- '--install-hive',
66
- '--hive-versions',
67
- 'latest'
67
+ '--base-path',
68
+ 's3://elasticmapreduce/libs/hive/',
69
+ '--install-hive',
70
+ '--hive-versions',
71
+ 'latest'
68
72
  ],
69
73
  },
70
74
  :name => 'Elasticity - Install Hive'
@@ -68,7 +68,11 @@ describe 'Elasticity::JobFlow Integration Examples' do
68
68
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
69
69
  :args => [
70
70
  's3://elasticmapreduce/libs/hive/hive-script',
71
- '--run-hive-script',
71
+ '--base-path',
72
+ 's3://elasticmapreduce/libs/hive/',
73
+ '--hive-versions',
74
+ 'latest',
75
+ '--run-hive-script',
72
76
  '--args',
73
77
  '-f', 's3n://slif-hive/test.q',
74
78
  '-d', 'OUTPUT=s3n://slif-test/output'
@@ -17,18 +17,19 @@ describe Elasticity::SyncToS3 do
17
17
  describe 'basic assignment' do
18
18
 
19
19
  it 'should set the proper values' do
20
- sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret')
20
+ sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret', 'region')
21
21
  sync.access_key.should == 'access'
22
22
  sync.secret_key.should == 'secret'
23
23
  sync.bucket_name.should == 'bucket'
24
+ sync.region.should == 'region'
24
25
  end
25
26
 
26
27
  end
27
28
 
28
- context 'when access and secret keys are nil' do
29
+ context 'when parameters are missing' do
29
30
 
30
- let(:both_keys_nil) { Elasticity::SyncToS3.new('_', nil, nil) }
31
- let(:both_keys_missing) { Elasticity::SyncToS3.new('_') }
31
+ let(:optional_params_nil) { Elasticity::SyncToS3.new('_', nil, nil, nil) }
32
+ let(:optional_params_missing) { Elasticity::SyncToS3.new('_') }
32
33
 
33
34
  before do
34
35
  ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return(access_key)
@@ -38,13 +39,21 @@ describe Elasticity::SyncToS3 do
38
39
  context 'when environment variables are present' do
39
40
  let(:access_key) { 'ENV_ACCESS' }
40
41
  let(:secret_key) { 'ENV_SECRET' }
41
- it 'should assign them to the keys' do
42
- both_keys_nil.access_key.should == 'ENV_ACCESS'
43
- both_keys_nil.secret_key.should == 'ENV_SECRET'
44
42
 
45
- both_keys_missing.access_key.should == 'ENV_ACCESS'
46
- both_keys_missing.secret_key.should == 'ENV_SECRET'
43
+ it 'should assign environment variables to the keys' do
44
+ optional_params_nil.access_key.should == 'ENV_ACCESS'
45
+ optional_params_nil.secret_key.should == 'ENV_SECRET'
46
+ optional_params_missing.access_key.should == 'ENV_ACCESS'
47
+ optional_params_missing.secret_key.should == 'ENV_SECRET'
47
48
  end
49
+
50
+ it 'should set the region to us-east-1' do
51
+ optional_params_nil.region.should == 'us-east-1'
52
+ optional_params_nil.region.should == 'us-east-1'
53
+ optional_params_missing.region.should == 'us-east-1'
54
+ optional_params_missing.region.should == 'us-east-1'
55
+ end
56
+
48
57
  end
49
58
 
50
59
  context 'when environment variables are not present' do
@@ -54,7 +63,7 @@ describe Elasticity::SyncToS3 do
54
63
  let(:secret_key) { '_' }
55
64
  it 'should raise an error' do
56
65
  expect {
57
- both_keys_nil # Trigger instantiation
66
+ optional_params_nil # Trigger instantiation
58
67
  }.to raise_error(Elasticity::MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.')
59
68
  end
60
69
  end
@@ -64,7 +73,7 @@ describe Elasticity::SyncToS3 do
64
73
  let(:secret_key) { nil }
65
74
  it 'should raise an error' do
66
75
  expect {
67
- both_keys_nil # Trigger instantiation
76
+ optional_params_nil # Trigger instantiation
68
77
  }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
69
78
  end
70
79
  end
@@ -77,50 +86,23 @@ describe Elasticity::SyncToS3 do
77
86
 
78
87
  describe '#sync' do
79
88
 
80
- context 'when the bucket exists' do
81
-
82
- before do
83
- s3.directories.create(:key => bucket_name)
84
- end
85
-
86
- context 'when the local directory exists' do
87
- before do
88
- FileUtils.mkdir('GOOD_DIR')
89
- end
90
- it 'should sync that directory' do
91
- sync_to_s3.should_receive(:sync_dir).with('GOOD_DIR', 'REMOTE_DIR')
92
- sync_to_s3.sync('GOOD_DIR', 'REMOTE_DIR')
93
- end
94
- end
95
-
96
- context 'when the local directory does not exist' do
97
- it 'should raise an error' do
98
- expect {
99
- sync_to_s3.sync('BAD_DIR', '_')
100
- }.to raise_error(Elasticity::NoDirectoryError, "Directory 'BAD_DIR' does not exist or is not a directory")
101
- end
102
- end
103
-
104
- context 'when the local directory is not a directory' do
105
- before do
106
- FileUtils.touch('NOT_A_DIR')
107
- end
108
- it 'should raise an error' do
109
- expect {
110
- sync_to_s3.sync('NOT_A_DIR', '_')
111
- }.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
112
- end
113
- end
89
+ let(:dir_name) { 'GOOD_DIR' }
90
+ let(:file_name) { File.join(%w(GOOD_DIR file_1)) }
91
+ let(:remote_dir) { 'REMOTE_DIR' }
114
92
 
93
+ before do
94
+ FileUtils.mkdir(dir_name)
95
+ FileUtils.touch(file_name)
96
+ end
97
+
98
+ it 'should sync directories' do
99
+ sync_to_s3.should_receive(:sync_dir).with(dir_name, remote_dir)
100
+ sync_to_s3.sync(dir_name, remote_dir)
115
101
  end
116
102
 
117
- context 'when the bucket does not exist' do
118
- let(:bucket_name) { 'BAD_BUCKET' }
119
- it 'should raise an error' do
120
- expect {
121
- sync_to_s3.sync('_', '_')
122
- }.to raise_error(Elasticity::NoBucketError, "Bucket 'BAD_BUCKET' does not exist")
123
- end
103
+ it 'should sync files' do
104
+ sync_to_s3.should_receive(:sync_file).with(file_name, remote_dir)
105
+ sync_to_s3.sync(file_name, remote_dir)
124
106
  end
125
107
 
126
108
  end
@@ -156,6 +138,14 @@ describe Elasticity::SyncToS3 do
156
138
  end
157
139
  end
158
140
 
141
+ context 'when the directory does not exist or is not a directory' do
142
+ it 'should raise an error' do
143
+ expect {
144
+ sync_to_s3.send(:sync_dir, 'NOT_A_DIR', '_')
145
+ }.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
146
+ end
147
+ end
148
+
159
149
  end
160
150
 
161
151
  describe '#sync_file' do
@@ -164,13 +154,13 @@ describe Elasticity::SyncToS3 do
164
154
  let(:file_name) { 'test.out' }
165
155
  let(:full_path) { File.join([local_dir, file_name]) }
166
156
  let(:remote_dir) { 'job/assets' }
167
- let(:remote_path) { "#{remote_dir}/#{file_name}"}
157
+ let(:remote_path) { "#{remote_dir}/#{file_name}" }
168
158
  let(:file_data) { 'Some test content' }
169
159
 
170
160
  before do
171
161
  s3.directories.create(:key => bucket_name)
172
162
  FileUtils.makedirs(local_dir)
173
- File.open(full_path, 'w') {|f| f.write(file_data) }
163
+ File.open(full_path, 'w') { |f| f.write(file_data) }
174
164
  end
175
165
 
176
166
  it 'should write the specified file into the remote directory' do
@@ -202,34 +192,56 @@ describe Elasticity::SyncToS3 do
202
192
  end
203
193
 
204
194
  context 'when remote dir is empty' do
205
- let(:remote_dir) {''}
195
+ let(:remote_dir) { '' }
206
196
  it 'should place files in the root without a bunk empty folder name' do
207
197
  s3.directories[0].files.head(file_name).should_not be_nil
208
198
  end
209
199
  end
210
200
 
211
201
  context 'when remote dir is /' do
212
- let(:remote_dir) {'/'}
202
+ let(:remote_dir) { '/' }
213
203
  it 'should place files in the root without a bunk empty folder name' do
214
204
  s3.directories[0].files.head(file_name).should_not be_nil
215
205
  end
216
206
  end
217
207
 
218
208
  context 'when remote dir starts with a /' do
219
- let(:remote_dir) {'/starts_with_slash'}
209
+ let(:remote_dir) { '/starts_with_slash' }
220
210
  it 'should place files in the root without a bunk empty folder name' do
221
211
  s3.directories[0].files.head('starts_with_slash/test.out').should_not be_nil
222
212
  end
223
213
  end
224
214
  end
225
215
 
216
+ context 'when the file does not exist' do
217
+ it 'should raise an error' do
218
+ expect {
219
+ sync_to_s3.send(:sync_file, 'NO_FILE', '_')
220
+ }.to raise_error(Elasticity::NoFileError, "File 'NO_FILE' does not exist")
221
+ end
222
+ end
223
+
224
+ context 'when the bucket does not exist' do
225
+ let(:no_bucket_sync) { Elasticity::SyncToS3.new('NEW_BUCKET', '_', '_') }
226
+ before do
227
+ no_bucket_sync.stub(:s3).and_return(s3)
228
+ FileUtils.mkdir('GOOD_DIR')
229
+ FileUtils.touch(File.join(%w(GOOD_DIR file_1)))
230
+ end
231
+ it 'should create the bucket in s3' do
232
+ no_bucket_sync.sync('GOOD_DIR', '_')
233
+ s3.directories.find { |d| d.key == 'NEW_BUCKET' }.should_not be_nil
234
+ end
235
+ end
236
+
226
237
  end
227
238
 
228
239
  describe '#s3' do
229
- let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret') }
240
+ let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret', 'region') }
230
241
  it 'should connect to S3 using the specified credentials' do
231
242
  Fog::Storage.should_receive(:new).with({
232
243
  :provider => 'AWS',
244
+ :region => 'region',
233
245
  :aws_access_key_id => 'access',
234
246
  :aws_secret_access_key => 'secret'
235
247
  }).and_return('GOOD_CONNECTION')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticity
3
3
  version: !ruby/object:Gem::Version
4
- version: '2.4'
4
+ version: '2.5'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-02 00:00:00.000000000 Z
12
+ date: 2012-09-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rest-client
@@ -64,17 +64,17 @@ dependencies:
64
64
  requirement: !ruby/object:Gem::Requirement
65
65
  none: false
66
66
  requirements:
67
- - - ! '>='
67
+ - - ~>
68
68
  - !ruby/object:Gem::Version
69
- version: '0'
69
+ version: '0.9'
70
70
  type: :development
71
71
  prerelease: false
72
72
  version_requirements: !ruby/object:Gem::Requirement
73
73
  none: false
74
74
  requirements:
75
- - - ! '>='
75
+ - - ~>
76
76
  - !ruby/object:Gem::Version
77
- version: '0'
77
+ version: '0.9'
78
78
  - !ruby/object:Gem::Dependency
79
79
  name: rspec
80
80
  requirement: !ruby/object:Gem::Requirement
@@ -155,7 +155,6 @@ files:
155
155
  - lib/elasticity/job_flow_step.rb
156
156
  - lib/elasticity/pig_step.rb
157
157
  - lib/elasticity/streaming_step.rb
158
- - lib/elasticity/support/conditional_raise.rb
159
158
  - lib/elasticity/sync_to_s3.rb
160
159
  - lib/elasticity/version.rb
161
160
  - spec/lib/elasticity/aws_request_spec.rb
@@ -173,7 +172,6 @@ files:
173
172
  - spec/lib/elasticity/job_flow_step_spec.rb
174
173
  - spec/lib/elasticity/pig_step_spec.rb
175
174
  - spec/lib/elasticity/streaming_step_spec.rb
176
- - spec/lib/elasticity/support/conditional_raise_spec.rb
177
175
  - spec/lib/elasticity/sync_to_s3_spec.rb
178
176
  - spec/spec_helper.rb
179
177
  - spec/support/be_a_hash_including_matcher.rb
@@ -191,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
191
189
  version: '0'
192
190
  segments:
193
191
  - 0
194
- hash: -2552375759855937320
192
+ hash: -4541363325950914992
195
193
  required_rubygems_version: !ruby/object:Gem::Requirement
196
194
  none: false
197
195
  requirements:
@@ -200,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
200
198
  version: '0'
201
199
  segments:
202
200
  - 0
203
- hash: -2552375759855937320
201
+ hash: -4541363325950914992
204
202
  requirements: []
205
203
  rubyforge_project:
206
204
  rubygems_version: 1.8.24
@@ -223,7 +221,6 @@ test_files:
223
221
  - spec/lib/elasticity/job_flow_step_spec.rb
224
222
  - spec/lib/elasticity/pig_step_spec.rb
225
223
  - spec/lib/elasticity/streaming_step_spec.rb
226
- - spec/lib/elasticity/support/conditional_raise_spec.rb
227
224
  - spec/lib/elasticity/sync_to_s3_spec.rb
228
225
  - spec/spec_helper.rb
229
226
  - spec/support/be_a_hash_including_matcher.rb
@@ -1,23 +0,0 @@
1
- module Elasticity
2
-
3
- module ConditionalRaising
4
-
5
- def raise_if(conditional, error_class, message)
6
- raise error_class, message if conditional
7
- end
8
-
9
- def raise_unless(conditional, error_class, message)
10
- raise error_class, message unless conditional
11
- end
12
-
13
- end
14
-
15
- end
16
-
17
- module Kernel
18
- include Elasticity::ConditionalRaising
19
- end
20
-
21
- class Object
22
- include Kernel
23
- end
@@ -1,35 +0,0 @@
1
- describe 'Conditional Raising' do
2
-
3
- describe '#raise_if' do
4
-
5
- it 'should raise the specified error if the condition is true' do
6
- expect {
7
- raise_if true, RuntimeError, 'MESSAGE'
8
- }.to raise_error(RuntimeError, 'MESSAGE')
9
- end
10
-
11
- it 'should not raise the specified error if the condition is false' do
12
- expect {
13
- raise_if false, RuntimeError, 'MESSAGE'
14
- }.to_not raise_error(RuntimeError, 'MESSAGE')
15
- end
16
-
17
- end
18
-
19
- describe '#raise_unless' do
20
-
21
- it 'should not raise the specified error unless the condition is true' do
22
- expect {
23
- raise_unless true, RuntimeError, 'MESSAGE'
24
- }.to_not raise_error(RuntimeError, 'MESSAGE')
25
- end
26
-
27
- it 'should raise the specified error unless the condition is false' do
28
- expect {
29
- raise_unless false, RuntimeError, 'MESSAGE'
30
- }.to raise_error(RuntimeError, 'MESSAGE')
31
- end
32
-
33
- end
34
-
35
- end