elasticity 2.4 → 2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/HISTORY.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 2.5 - September 29, 2012
2
+
3
+ + ```SyncToS3``` supports S3 region specification.
4
+ + ```SyncToS3#sync``` supports being called with both files and directories.
5
+
1
6
  ## 2.4 - September 1, 2012
2
7
 
3
8
  + ```SyncToS3``` added to enable one-way asset synchronization.
data/README.md CHANGED
@@ -16,7 +16,7 @@ gem install elasticity
16
16
  or in your Gemfile
17
17
 
18
18
  ```
19
- gem 'elasticity', '~> 2.0'
19
+ gem 'elasticity', '~> 2.5'
20
20
  ```
21
21
 
22
22
  This will ensure that you protect yourself from API changes, which will only be made in major revisions.
@@ -273,20 +273,27 @@ jobflow.add_step(jar_step)
273
273
 
274
274
  ## 6 - Upload Assets (optional)
275
275
 
276
- This isn't part of ```JobFlow```; more of an aside :) Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. For example, a TSV file with a range of valid values, join tables, etc.
276
+ This isn't part of ```JobFlow```; more of an aside. Elasticity provides a very basic means of uploading assets to S3 so that your EMR job has access to them. Most commonly this will be a set of resources to run the job (e.g. JAR files, streaming scripts, etc.) and a set of resources used by the job itself (e.g. a TSV file with a range of valid values, join tables, etc.).
277
277
 
278
278
  ```ruby
279
- # Specify the bucket and AWS credentials
280
- s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret')
279
+ # Specify the bucket name, AWS credentials and region
280
+ s3 = Elasticity::SyncToS3('my-bucket', 'access', 'secret', 'region')
281
281
 
282
- # Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
282
+ # Alternatively, specify nothing :)
283
+ # - Use the standard environment variables (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
284
+ # - Use the 'us-east-1' region by default
283
285
  # s3 = Elasticity::SyncToS3('my-bucket')
284
286
 
285
- # Recursively sync the contents of '/some/parent/dir' under the remote location 'remote-dir/this-job/assets'
286
- s3.sync('/some/parent/dir', 'remote-dir/this-job/assets')
287
+ # Recursively sync the contents of '/foo' under the remote location 'remote-dir/this-job'
288
+ s3.sync('/foo', 'remote-dir/this-job')
289
+
290
+ # Sync a single file to a remote directory
291
+ s3.sync('/foo/this-job/tables/join.tsv', 'remote-dir/this-job/tables')
287
292
  ```
288
293
 
289
- If the files already exist, there is an MD5 checksum check. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/assets/join.tsv``` in your EMR jobs.
294
+ If the bucket doesn't exist, it will be created.
295
+
296
+ If a file already exists, there is an MD5 checksum evaluation. If the checksums are the same, the file will be skipped. Now you can use something like ```s3n://my-bucket/remote-dir/this-job/tables/join.tsv``` in your EMR jobs.
290
297
 
291
298
  ## 7 - Run the Job Flow
292
299
 
data/elasticity.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
15
15
  s.add_dependency('nokogiri')
16
16
  s.add_dependency('fog')
17
17
 
18
- s.add_development_dependency('rake')
18
+ s.add_development_dependency('rake', '~> 0.9')
19
19
  s.add_development_dependency('rspec', '~> 2.11.0')
20
20
  s.add_development_dependency('timecop')
21
21
  s.add_development_dependency('fakefs', '~> 0.4')
data/lib/elasticity.rb CHANGED
@@ -5,8 +5,6 @@ require 'rest_client'
5
5
  require 'nokogiri'
6
6
  require 'fog'
7
7
 
8
- require 'elasticity/support/conditional_raise'
9
-
10
8
  require 'elasticity/aws_request'
11
9
  require 'elasticity/emr'
12
10
 
@@ -50,7 +50,7 @@ module Elasticity
50
50
  def get_secret_key(secret)
51
51
  return secret if secret
52
52
  return ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
53
- raise MissingKeyError, 'Please provide a secret key or set AWS_ACCESS_KEY_ID.'
53
+ raise MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.'
54
54
  end
55
55
 
56
56
  # (Used from RightScale's right_aws gem.)
@@ -17,7 +17,7 @@ module Elasticity
17
17
  end
18
18
 
19
19
  def to_aws_step(job_flow)
20
- args = %w(s3://elasticmapreduce/libs/hive/hive-script --run-hive-script --args)
20
+ args = %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions latest --run-hive-script --args)
21
21
  args.concat(['-f', @script])
22
22
  @variables.keys.sort.each do |name|
23
23
  args.concat(['-d', "#{name}=#{@variables[name]}"])
@@ -43,11 +43,11 @@ module Elasticity
43
43
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
44
44
  :args => [
45
45
  's3://elasticmapreduce/libs/hive/hive-script',
46
- '--base-path',
47
- 's3://elasticmapreduce/libs/hive/',
48
- '--install-hive',
49
- '--hive-versions',
50
- 'latest'
46
+ '--base-path',
47
+ 's3://elasticmapreduce/libs/hive/',
48
+ '--install-hive',
49
+ '--hive-versions',
50
+ 'latest'
51
51
  ],
52
52
  },
53
53
  :name => 'Elasticity - Install Hive'
@@ -19,19 +19,27 @@ module Elasticity
19
19
  end
20
20
 
21
21
  def count=(instance_count)
22
- raise_if instance_count <= 0, ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
23
- raise_if @role == 'MASTER' && instance_count != 1, ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
22
+ if instance_count <= 0
23
+ raise ArgumentError, "Instance groups require at least 1 instance (#{instance_count} requested)"
24
+ end
25
+ if @role == 'MASTER' && instance_count != 1
26
+ raise ArgumentError, "MASTER instance groups can only have 1 instance (#{instance_count} requested)"
27
+ end
24
28
  @count = instance_count
25
29
  end
26
30
 
27
31
  def role=(group_role)
28
- raise_unless ROLES.include?(group_role), ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
32
+ if !ROLES.include?(group_role)
33
+ raise ArgumentError, "Role must be one of MASTER, CORE or TASK (#{group_role} was requested)"
34
+ end
29
35
  @count = 1 if group_role == 'MASTER'
30
36
  @role = group_role
31
37
  end
32
38
 
33
39
  def set_spot_instances(bid_price)
34
- raise_unless bid_price > 0, ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
40
+ if bid_price < 0
41
+ raise ArgumentError, "The bid price for spot instances should be greater than 0 (#{bid_price} requested)"
42
+ end
35
43
  @bid_price = bid_price
36
44
  @market = 'SPOT'
37
45
  end
@@ -21,7 +21,7 @@ module Elasticity
21
21
 
22
22
  attr_reader :access_key
23
23
  attr_reader :secret_key
24
-
24
+
25
25
  def initialize(access=nil, secret=nil)
26
26
  @action_on_failure = 'TERMINATE_JOB_FLOW'
27
27
  @hadoop_version = '1.0.3'
@@ -73,7 +73,9 @@ module Elasticity
73
73
  end
74
74
 
75
75
  def add_bootstrap_action(bootstrap_action)
76
- raise_if is_jobflow_running?, JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
76
+ if is_jobflow_running?
77
+ raise JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
78
+ end
77
79
  @bootstrap_actions << bootstrap_action
78
80
  end
79
81
 
@@ -106,18 +108,26 @@ module Elasticity
106
108
  end
107
109
 
108
110
  def run
109
- raise_if @jobflow_steps.empty?, JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
110
- raise_if is_jobflow_running?, JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
111
+ if @jobflow_steps.empty?
112
+ raise JobFlowMissingStepsError, 'Cannot run a job flow without adding steps. Please use #add_step.'
113
+ end
114
+ if is_jobflow_running?
115
+ raise JobFlowRunningError, 'Cannot run a job flow multiple times. To do more with this job flow, please use #add_step.'
116
+ end
111
117
  @jobflow_id = emr.run_job_flow(jobflow_config)
112
118
  end
113
119
 
114
120
  def shutdown
115
- raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
121
+ if !is_jobflow_running?
122
+ raise JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
123
+ end
116
124
  emr.terminate_jobflows(@jobflow_id)
117
125
  end
118
126
 
119
127
  def status
120
- raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
128
+ if !is_jobflow_running?
129
+ raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
130
+ end
121
131
  emr.describe_jobflow(@jobflow_id)
122
132
  end
123
133
 
@@ -2,38 +2,45 @@ module Elasticity
2
2
 
3
3
  class NoBucketError < StandardError; end
4
4
  class NoDirectoryError < StandardError; end
5
+ class NoFileError < StandardError; end
5
6
 
6
7
  class SyncToS3
7
8
 
8
9
  attr_reader :access_key
9
10
  attr_reader :secret_key
10
11
  attr_reader :bucket_name
12
+ attr_reader :region
11
13
 
12
- def initialize(bucket, access=nil, secret=nil)
14
+ def initialize(bucket, access=nil, secret=nil, region=nil)
13
15
  @access_key = get_access_key(access)
14
16
  @secret_key = get_secret_key(secret)
15
17
  @bucket_name = bucket
18
+ @region = region ||= 'us-east-1'
16
19
  end
17
20
 
18
21
  def sync(local, remote)
19
- raise_unless bucket, NoBucketError, "Bucket '#@bucket_name' does not exist"
20
- raise_unless File.directory?(local), NoDirectoryError, "Directory '#{local}' does not exist or is not a directory"
21
- sync_dir(local, remote)
22
+ if File.directory?(local)
23
+ sync_dir(local, remote)
24
+ else
25
+ sync_file(local, remote)
26
+ end
22
27
  end
23
28
 
24
29
  private
25
30
 
26
- def sync_dir(local, remote)
27
- Dir.glob(File.join([local, '*'])).each do |entry|
31
+ def sync_dir(dir_name, remote_dir)
32
+ raise NoDirectoryError, "Directory '#{dir_name}' does not exist or is not a directory" unless File.directory?(dir_name)
33
+ Dir.glob(File.join([dir_name, '*'])).each do |entry|
28
34
  if File.directory?(entry)
29
- sync_dir(entry, [remote, File.basename(entry)].join('/'))
35
+ sync_dir(entry, [remote_dir, File.basename(entry)].join('/'))
30
36
  else
31
- sync_file(entry, remote)
37
+ sync_file(entry, remote_dir)
32
38
  end
33
39
  end
34
40
  end
35
41
 
36
42
  def sync_file(file_name, remote_dir)
43
+ raise NoFileError, "File '#{file_name}' does not exist" unless File.exists?(file_name)
37
44
  remote_dir = remote_dir.gsub(/^(\/)/, '')
38
45
  remote_path = (remote_dir.empty?) ? (File.basename(file_name)) : [remote_dir, File.basename(file_name)].join('/')
39
46
  metadata = bucket.files.head(remote_path)
@@ -46,14 +53,17 @@ module Elasticity
46
53
  })
47
54
  end
48
55
 
56
+ private
57
+
49
58
  def bucket
50
- index = s3.directories.index { |d| d.key == @bucket_name }
51
- @bucket ||= index ? s3.directories[index] : nil
59
+ @bucket ||= s3.directories.find { |d| d.key == @bucket_name }
60
+ @bucket ||= s3.directories.create(:key => @bucket_name)
52
61
  end
53
62
 
54
63
  def s3
55
64
  @connection ||= Fog::Storage.new({
56
65
  :provider => 'AWS',
66
+ :region => @region,
57
67
  :aws_access_key_id => @access_key,
58
68
  :aws_secret_access_key => @secret_key
59
69
  })
@@ -1,3 +1,3 @@
1
1
  module Elasticity
2
- VERSION = '2.4'
2
+ VERSION = '2.5'
3
3
  end
@@ -68,7 +68,7 @@ describe Elasticity::AwsRequest do
68
68
  it 'should raise an error' do
69
69
  expect {
70
70
  missing_something.access_key
71
- }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_ACCESS_KEY_ID.')
71
+ }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
72
72
  end
73
73
  end
74
74
  end
@@ -20,10 +20,14 @@ describe Elasticity::HiveStep do
20
20
  step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
21
21
  step[:hadoop_jar_step][:args].should start_with([
22
22
  's3://elasticmapreduce/libs/hive/hive-script',
23
- '--run-hive-script',
24
- '--args',
25
- '-f',
26
- 'script.hql'
23
+ '--base-path',
24
+ 's3://elasticmapreduce/libs/hive/',
25
+ '--hive-versions',
26
+ 'latest',
27
+ '--run-hive-script',
28
+ '--args',
29
+ '-f',
30
+ 'script.hql'
27
31
  ])
28
32
  end
29
33
 
@@ -39,7 +43,7 @@ describe Elasticity::HiveStep do
39
43
 
40
44
  it 'should convert to aws step format' do
41
45
  step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
42
- step[:hadoop_jar_step][:args][5..9].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
46
+ step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
43
47
  end
44
48
  end
45
49
 
@@ -60,11 +64,11 @@ describe Elasticity::HiveStep do
60
64
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
61
65
  :args => [
62
66
  's3://elasticmapreduce/libs/hive/hive-script',
63
- '--base-path',
64
- 's3://elasticmapreduce/libs/hive/',
65
- '--install-hive',
66
- '--hive-versions',
67
- 'latest'
67
+ '--base-path',
68
+ 's3://elasticmapreduce/libs/hive/',
69
+ '--install-hive',
70
+ '--hive-versions',
71
+ 'latest'
68
72
  ],
69
73
  },
70
74
  :name => 'Elasticity - Install Hive'
@@ -68,7 +68,11 @@ describe 'Elasticity::JobFlow Integration Examples' do
68
68
  :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
69
69
  :args => [
70
70
  's3://elasticmapreduce/libs/hive/hive-script',
71
- '--run-hive-script',
71
+ '--base-path',
72
+ 's3://elasticmapreduce/libs/hive/',
73
+ '--hive-versions',
74
+ 'latest',
75
+ '--run-hive-script',
72
76
  '--args',
73
77
  '-f', 's3n://slif-hive/test.q',
74
78
  '-d', 'OUTPUT=s3n://slif-test/output'
@@ -17,18 +17,19 @@ describe Elasticity::SyncToS3 do
17
17
  describe 'basic assignment' do
18
18
 
19
19
  it 'should set the proper values' do
20
- sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret')
20
+ sync = Elasticity::SyncToS3.new('bucket', 'access', 'secret', 'region')
21
21
  sync.access_key.should == 'access'
22
22
  sync.secret_key.should == 'secret'
23
23
  sync.bucket_name.should == 'bucket'
24
+ sync.region.should == 'region'
24
25
  end
25
26
 
26
27
  end
27
28
 
28
- context 'when access and secret keys are nil' do
29
+ context 'when parameters are missing' do
29
30
 
30
- let(:both_keys_nil) { Elasticity::SyncToS3.new('_', nil, nil) }
31
- let(:both_keys_missing) { Elasticity::SyncToS3.new('_') }
31
+ let(:optional_params_nil) { Elasticity::SyncToS3.new('_', nil, nil, nil) }
32
+ let(:optional_params_missing) { Elasticity::SyncToS3.new('_') }
32
33
 
33
34
  before do
34
35
  ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return(access_key)
@@ -38,13 +39,21 @@ describe Elasticity::SyncToS3 do
38
39
  context 'when environment variables are present' do
39
40
  let(:access_key) { 'ENV_ACCESS' }
40
41
  let(:secret_key) { 'ENV_SECRET' }
41
- it 'should assign them to the keys' do
42
- both_keys_nil.access_key.should == 'ENV_ACCESS'
43
- both_keys_nil.secret_key.should == 'ENV_SECRET'
44
42
 
45
- both_keys_missing.access_key.should == 'ENV_ACCESS'
46
- both_keys_missing.secret_key.should == 'ENV_SECRET'
43
+ it 'should assign environment variables to the keys' do
44
+ optional_params_nil.access_key.should == 'ENV_ACCESS'
45
+ optional_params_nil.secret_key.should == 'ENV_SECRET'
46
+ optional_params_missing.access_key.should == 'ENV_ACCESS'
47
+ optional_params_missing.secret_key.should == 'ENV_SECRET'
47
48
  end
49
+
50
+ it 'should set the region to us-east-1' do
51
+ optional_params_nil.region.should == 'us-east-1'
52
+ optional_params_nil.region.should == 'us-east-1'
53
+ optional_params_missing.region.should == 'us-east-1'
54
+ optional_params_missing.region.should == 'us-east-1'
55
+ end
56
+
48
57
  end
49
58
 
50
59
  context 'when environment variables are not present' do
@@ -54,7 +63,7 @@ describe Elasticity::SyncToS3 do
54
63
  let(:secret_key) { '_' }
55
64
  it 'should raise an error' do
56
65
  expect {
57
- both_keys_nil # Trigger instantiation
66
+ optional_params_nil # Trigger instantiation
58
67
  }.to raise_error(Elasticity::MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.')
59
68
  end
60
69
  end
@@ -64,7 +73,7 @@ describe Elasticity::SyncToS3 do
64
73
  let(:secret_key) { nil }
65
74
  it 'should raise an error' do
66
75
  expect {
67
- both_keys_nil # Trigger instantiation
76
+ optional_params_nil # Trigger instantiation
68
77
  }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
69
78
  end
70
79
  end
@@ -77,50 +86,23 @@ describe Elasticity::SyncToS3 do
77
86
 
78
87
  describe '#sync' do
79
88
 
80
- context 'when the bucket exists' do
81
-
82
- before do
83
- s3.directories.create(:key => bucket_name)
84
- end
85
-
86
- context 'when the local directory exists' do
87
- before do
88
- FileUtils.mkdir('GOOD_DIR')
89
- end
90
- it 'should sync that directory' do
91
- sync_to_s3.should_receive(:sync_dir).with('GOOD_DIR', 'REMOTE_DIR')
92
- sync_to_s3.sync('GOOD_DIR', 'REMOTE_DIR')
93
- end
94
- end
95
-
96
- context 'when the local directory does not exist' do
97
- it 'should raise an error' do
98
- expect {
99
- sync_to_s3.sync('BAD_DIR', '_')
100
- }.to raise_error(Elasticity::NoDirectoryError, "Directory 'BAD_DIR' does not exist or is not a directory")
101
- end
102
- end
103
-
104
- context 'when the local directory is not a directory' do
105
- before do
106
- FileUtils.touch('NOT_A_DIR')
107
- end
108
- it 'should raise an error' do
109
- expect {
110
- sync_to_s3.sync('NOT_A_DIR', '_')
111
- }.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
112
- end
113
- end
89
+ let(:dir_name) { 'GOOD_DIR' }
90
+ let(:file_name) { File.join(%w(GOOD_DIR file_1)) }
91
+ let(:remote_dir) { 'REMOTE_DIR' }
114
92
 
93
+ before do
94
+ FileUtils.mkdir(dir_name)
95
+ FileUtils.touch(file_name)
96
+ end
97
+
98
+ it 'should sync directories' do
99
+ sync_to_s3.should_receive(:sync_dir).with(dir_name, remote_dir)
100
+ sync_to_s3.sync(dir_name, remote_dir)
115
101
  end
116
102
 
117
- context 'when the bucket does not exist' do
118
- let(:bucket_name) { 'BAD_BUCKET' }
119
- it 'should raise an error' do
120
- expect {
121
- sync_to_s3.sync('_', '_')
122
- }.to raise_error(Elasticity::NoBucketError, "Bucket 'BAD_BUCKET' does not exist")
123
- end
103
+ it 'should sync files' do
104
+ sync_to_s3.should_receive(:sync_file).with(file_name, remote_dir)
105
+ sync_to_s3.sync(file_name, remote_dir)
124
106
  end
125
107
 
126
108
  end
@@ -156,6 +138,14 @@ describe Elasticity::SyncToS3 do
156
138
  end
157
139
  end
158
140
 
141
+ context 'when the directory does not exist or is not a directory' do
142
+ it 'should raise an error' do
143
+ expect {
144
+ sync_to_s3.send(:sync_dir, 'NOT_A_DIR', '_')
145
+ }.to raise_error(Elasticity::NoDirectoryError, "Directory 'NOT_A_DIR' does not exist or is not a directory")
146
+ end
147
+ end
148
+
159
149
  end
160
150
 
161
151
  describe '#sync_file' do
@@ -164,13 +154,13 @@ describe Elasticity::SyncToS3 do
164
154
  let(:file_name) { 'test.out' }
165
155
  let(:full_path) { File.join([local_dir, file_name]) }
166
156
  let(:remote_dir) { 'job/assets' }
167
- let(:remote_path) { "#{remote_dir}/#{file_name}"}
157
+ let(:remote_path) { "#{remote_dir}/#{file_name}" }
168
158
  let(:file_data) { 'Some test content' }
169
159
 
170
160
  before do
171
161
  s3.directories.create(:key => bucket_name)
172
162
  FileUtils.makedirs(local_dir)
173
- File.open(full_path, 'w') {|f| f.write(file_data) }
163
+ File.open(full_path, 'w') { |f| f.write(file_data) }
174
164
  end
175
165
 
176
166
  it 'should write the specified file into the remote directory' do
@@ -202,34 +192,56 @@ describe Elasticity::SyncToS3 do
202
192
  end
203
193
 
204
194
  context 'when remote dir is empty' do
205
- let(:remote_dir) {''}
195
+ let(:remote_dir) { '' }
206
196
  it 'should place files in the root without a bunk empty folder name' do
207
197
  s3.directories[0].files.head(file_name).should_not be_nil
208
198
  end
209
199
  end
210
200
 
211
201
  context 'when remote dir is /' do
212
- let(:remote_dir) {'/'}
202
+ let(:remote_dir) { '/' }
213
203
  it 'should place files in the root without a bunk empty folder name' do
214
204
  s3.directories[0].files.head(file_name).should_not be_nil
215
205
  end
216
206
  end
217
207
 
218
208
  context 'when remote dir starts with a /' do
219
- let(:remote_dir) {'/starts_with_slash'}
209
+ let(:remote_dir) { '/starts_with_slash' }
220
210
  it 'should place files in the root without a bunk empty folder name' do
221
211
  s3.directories[0].files.head('starts_with_slash/test.out').should_not be_nil
222
212
  end
223
213
  end
224
214
  end
225
215
 
216
+ context 'when the file does not exist' do
217
+ it 'should raise an error' do
218
+ expect {
219
+ sync_to_s3.send(:sync_file, 'NO_FILE', '_')
220
+ }.to raise_error(Elasticity::NoFileError, "File 'NO_FILE' does not exist")
221
+ end
222
+ end
223
+
224
+ context 'when the bucket does not exist' do
225
+ let(:no_bucket_sync) { Elasticity::SyncToS3.new('NEW_BUCKET', '_', '_') }
226
+ before do
227
+ no_bucket_sync.stub(:s3).and_return(s3)
228
+ FileUtils.mkdir('GOOD_DIR')
229
+ FileUtils.touch(File.join(%w(GOOD_DIR file_1)))
230
+ end
231
+ it 'should create the bucket in s3' do
232
+ no_bucket_sync.sync('GOOD_DIR', '_')
233
+ s3.directories.find { |d| d.key == 'NEW_BUCKET' }.should_not be_nil
234
+ end
235
+ end
236
+
226
237
  end
227
238
 
228
239
  describe '#s3' do
229
- let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret') }
240
+ let(:connection_test) { Elasticity::SyncToS3.new('_', 'access', 'secret', 'region') }
230
241
  it 'should connect to S3 using the specified credentials' do
231
242
  Fog::Storage.should_receive(:new).with({
232
243
  :provider => 'AWS',
244
+ :region => 'region',
233
245
  :aws_access_key_id => 'access',
234
246
  :aws_secret_access_key => 'secret'
235
247
  }).and_return('GOOD_CONNECTION')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticity
3
3
  version: !ruby/object:Gem::Version
4
- version: '2.4'
4
+ version: '2.5'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-02 00:00:00.000000000 Z
12
+ date: 2012-09-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rest-client
@@ -64,17 +64,17 @@ dependencies:
64
64
  requirement: !ruby/object:Gem::Requirement
65
65
  none: false
66
66
  requirements:
67
- - - ! '>='
67
+ - - ~>
68
68
  - !ruby/object:Gem::Version
69
- version: '0'
69
+ version: '0.9'
70
70
  type: :development
71
71
  prerelease: false
72
72
  version_requirements: !ruby/object:Gem::Requirement
73
73
  none: false
74
74
  requirements:
75
- - - ! '>='
75
+ - - ~>
76
76
  - !ruby/object:Gem::Version
77
- version: '0'
77
+ version: '0.9'
78
78
  - !ruby/object:Gem::Dependency
79
79
  name: rspec
80
80
  requirement: !ruby/object:Gem::Requirement
@@ -155,7 +155,6 @@ files:
155
155
  - lib/elasticity/job_flow_step.rb
156
156
  - lib/elasticity/pig_step.rb
157
157
  - lib/elasticity/streaming_step.rb
158
- - lib/elasticity/support/conditional_raise.rb
159
158
  - lib/elasticity/sync_to_s3.rb
160
159
  - lib/elasticity/version.rb
161
160
  - spec/lib/elasticity/aws_request_spec.rb
@@ -173,7 +172,6 @@ files:
173
172
  - spec/lib/elasticity/job_flow_step_spec.rb
174
173
  - spec/lib/elasticity/pig_step_spec.rb
175
174
  - spec/lib/elasticity/streaming_step_spec.rb
176
- - spec/lib/elasticity/support/conditional_raise_spec.rb
177
175
  - spec/lib/elasticity/sync_to_s3_spec.rb
178
176
  - spec/spec_helper.rb
179
177
  - spec/support/be_a_hash_including_matcher.rb
@@ -191,7 +189,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
191
189
  version: '0'
192
190
  segments:
193
191
  - 0
194
- hash: -2552375759855937320
192
+ hash: -4541363325950914992
195
193
  required_rubygems_version: !ruby/object:Gem::Requirement
196
194
  none: false
197
195
  requirements:
@@ -200,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
200
198
  version: '0'
201
199
  segments:
202
200
  - 0
203
- hash: -2552375759855937320
201
+ hash: -4541363325950914992
204
202
  requirements: []
205
203
  rubyforge_project:
206
204
  rubygems_version: 1.8.24
@@ -223,7 +221,6 @@ test_files:
223
221
  - spec/lib/elasticity/job_flow_step_spec.rb
224
222
  - spec/lib/elasticity/pig_step_spec.rb
225
223
  - spec/lib/elasticity/streaming_step_spec.rb
226
- - spec/lib/elasticity/support/conditional_raise_spec.rb
227
224
  - spec/lib/elasticity/sync_to_s3_spec.rb
228
225
  - spec/spec_helper.rb
229
226
  - spec/support/be_a_hash_including_matcher.rb
@@ -1,23 +0,0 @@
1
- module Elasticity
2
-
3
- module ConditionalRaising
4
-
5
- def raise_if(conditional, error_class, message)
6
- raise error_class, message if conditional
7
- end
8
-
9
- def raise_unless(conditional, error_class, message)
10
- raise error_class, message unless conditional
11
- end
12
-
13
- end
14
-
15
- end
16
-
17
- module Kernel
18
- include Elasticity::ConditionalRaising
19
- end
20
-
21
- class Object
22
- include Kernel
23
- end
@@ -1,35 +0,0 @@
1
- describe 'Conditional Raising' do
2
-
3
- describe '#raise_if' do
4
-
5
- it 'should raise the specified error if the condition is true' do
6
- expect {
7
- raise_if true, RuntimeError, 'MESSAGE'
8
- }.to raise_error(RuntimeError, 'MESSAGE')
9
- end
10
-
11
- it 'should not raise the specified error if the condition is false' do
12
- expect {
13
- raise_if false, RuntimeError, 'MESSAGE'
14
- }.to_not raise_error(RuntimeError, 'MESSAGE')
15
- end
16
-
17
- end
18
-
19
- describe '#raise_unless' do
20
-
21
- it 'should not raise the specified error unless the condition is true' do
22
- expect {
23
- raise_unless true, RuntimeError, 'MESSAGE'
24
- }.to_not raise_error(RuntimeError, 'MESSAGE')
25
- end
26
-
27
- it 'should raise the specified error unless the condition is false' do
28
- expect {
29
- raise_unless false, RuntimeError, 'MESSAGE'
30
- }.to raise_error(RuntimeError, 'MESSAGE')
31
- end
32
-
33
- end
34
-
35
- end