hasta 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -1
- data/hasta.gemspec +1 -1
- data/lib/hasta/emr_node.rb +2 -1
- data/lib/hasta/version.rb +1 -1
- data/spec/fixtures/hasta/json/emr_node.json +1 -1
- data/spec/hasta/emr_node_spec.rb +1 -0
- data/spec/hasta/local_storage_spec.rb +11 -1
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5dbce97906a4409f9621114228fd88a9d76ef11f
|
4
|
+
data.tar.gz: 077cf2b8bad29d39b688a02ba71fae387f4ca97e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 044dc81cd6ff504a15a00b51745de17a58b900f843fc8931af8572f4fe67eb770a3cb7a00ff03d8b7b749c56ef4bd79a4e38609ef14d10685ea604be9ef8502f
|
7
|
+
data.tar.gz: f60dd1d7cdb3b4ef275294c97cd91d4c7ec9a227f3c3132ab22b6bb647239f70da49f849a3ecf2f340caec319b1e549c305d51d112f321b6b255ddddb3cfa5f1
|
data/README.md
CHANGED
@@ -34,7 +34,8 @@ Or install it yourself as:
|
|
34
34
|
Hasta::Tasks::Runner.new do |task, opts|
|
35
35
|
task.definition_file = <path-to-AWS-datapipeline-definition-json-file>
|
36
36
|
task.job_id = opts[:job_id]
|
37
|
-
|
37
|
+
scheduled_start_time = opts[:scheduled_start_time]
|
38
|
+
task.scheduled_start_time = scheduled_start_time.nil? ? Time.now : Time.parse(scheduled_start_time)
|
38
39
|
task.project_root = File.dirname(__FILE__)
|
39
40
|
end
|
40
41
|
```
|
@@ -53,6 +54,7 @@ Or install it yourself as:
|
|
53
54
|
## Configuration
|
54
55
|
|
55
56
|
The following code snippet illustrates how to update the global Hasta configuration, which values are mandatory, and which values have defaults.
|
57
|
+
This should go in the Rakefile as well.
|
56
58
|
|
57
59
|
```ruby
|
58
60
|
Hasta.configure do |config|
|
data/hasta.gemspec
CHANGED
data/lib/hasta/emr_node.rb
CHANGED
@@ -85,7 +85,8 @@ module Hasta
|
|
85
85
|
end
|
86
86
|
|
87
87
|
def cache_files
|
88
|
-
|
88
|
+
files = attributes[:cache_files]
|
89
|
+
@cache_files ||= Hash[files.map { |value| interpolate(value).split('#').reverse }]
|
89
90
|
end
|
90
91
|
|
91
92
|
def env
|
data/lib/hasta/version.rb
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
"onFail": { "ref": "FailureNotify" },
|
5
5
|
"schedule": { "ref": "Nightly" },
|
6
6
|
"runsOn": { "ref": "MenuIntelEMRCluster" },
|
7
|
-
"step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
|
7
|
+
"step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cacheFile,s3://data-bucket/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/timestamped.yml#timestamped.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
|
8
8
|
"input": [ { "ref": "S3Input1" }, { "ref": "S3Input2" }, { "ref": "S3Input3" } ],
|
9
9
|
"output": { "ref": "S3Output" }
|
10
10
|
}
|
data/spec/hasta/emr_node_spec.rb
CHANGED
@@ -7,8 +7,18 @@ require 'hasta/local_file_path'
|
|
7
7
|
describe Hasta::LocalStorage do
|
8
8
|
subject { described_class.new(fog_storage) }
|
9
9
|
|
10
|
+
before do
|
11
|
+
Hasta.configure do |conf|
|
12
|
+
@original_storage_root = conf.local_storage_root
|
13
|
+
conf.local_storage_root = tmpdir
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
10
17
|
after do
|
11
18
|
FileUtils.rm_rf(tmpdir)
|
19
|
+
Hasta.configure do |conf|
|
20
|
+
conf.local_storage_root = @original_storage_root
|
21
|
+
end
|
12
22
|
end
|
13
23
|
|
14
24
|
let(:fog_storage) {
|
@@ -30,7 +40,7 @@ describe Hasta::LocalStorage do
|
|
30
40
|
let(:content) { "Hi\nBye\nWhy?\n" }
|
31
41
|
let(:data_source) { StringIO.new(content) }
|
32
42
|
|
33
|
-
let(:result) { subject.write(s3_uri, data_source) }
|
43
|
+
let!(:result) { subject.write(s3_uri, data_source) }
|
34
44
|
let(:local_file_path) { Hasta::LocalFilePath.for(result) }
|
35
45
|
|
36
46
|
context 'given a directory uri' do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hasta
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- danhodge
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-08-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fog
|
@@ -98,16 +98,16 @@ dependencies:
|
|
98
98
|
name: rspec
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - ~>
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
103
|
+
version: 2.13.0
|
104
104
|
type: :development
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - ~>
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
110
|
+
version: 2.13.0
|
111
111
|
description: Harness for locally testing streaming Hadoop jobs written in Ruby
|
112
112
|
email:
|
113
113
|
- dan@swipely.com
|