hasta 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 181a02eb6d37332af6efde332af20305f692f3e0
4
- data.tar.gz: b484f8ad5a8884b7f13e03caee7277f6b00d521b
3
+ metadata.gz: 5dbce97906a4409f9621114228fd88a9d76ef11f
4
+ data.tar.gz: 077cf2b8bad29d39b688a02ba71fae387f4ca97e
5
5
  SHA512:
6
- metadata.gz: a2206da5b8b905a66037ac9c23e7faab4c4a3254ef127988620bbe36eb6a04273677d8cf42785dacb889c0b7b35c9636549462a80ca6849f2b576839141b9756
7
- data.tar.gz: c7d96b9a9d4383e25156cad5ef58250b9713881eb70251b83347724264f08ef7adbe8ecdbc261f17c776bb12d6106466beb167a1284736fd8a0147a79d2bef25
6
+ metadata.gz: 044dc81cd6ff504a15a00b51745de17a58b900f843fc8931af8572f4fe67eb770a3cb7a00ff03d8b7b749c56ef4bd79a4e38609ef14d10685ea604be9ef8502f
7
+ data.tar.gz: f60dd1d7cdb3b4ef275294c97cd91d4c7ec9a227f3c3132ab22b6bb647239f70da49f849a3ecf2f340caec319b1e549c305d51d112f321b6b255ddddb3cfa5f1
data/README.md CHANGED
@@ -34,7 +34,8 @@ Or install it yourself as:
34
34
  Hasta::Tasks::Runner.new do |task, opts|
35
35
  task.definition_file = <path-to-AWS-datapipeline-definition-json-file>
36
36
  task.job_id = opts[:job_id]
37
- task.scheduled_start_time = Time.parse(opts[:scheduled_start_time])
37
+ scheduled_start_time = opts[:scheduled_start_time]
38
+ task.scheduled_start_time = scheduled_start_time.nil? ? Time.now : Time.parse(scheduled_start_time)
38
39
  task.project_root = File.dirname(__FILE__)
39
40
  end
40
41
  ```
@@ -53,6 +54,7 @@ Or install it yourself as:
53
54
  ## Configuration
54
55
 
55
56
  The following code snippet illustrates how to update the global Hasta configuration, which values are mandatory, and which values have defaults.
57
+ This should go in the Rakefile as well.
56
58
 
57
59
  ```ruby
58
60
  Hasta.configure do |config|
@@ -25,5 +25,5 @@ Gem::Specification.new do |spec|
25
25
  spec.add_development_dependency "cane"
26
26
  spec.add_development_dependency "pry"
27
27
  spec.add_development_dependency "rake"
28
- spec.add_development_dependency "rspec"
28
+ spec.add_development_dependency "rspec", '~> 2.13.0'
29
29
  end
@@ -85,7 +85,8 @@ module Hasta
85
85
  end
86
86
 
87
87
  def cache_files
88
- @cache_files ||= Hash[attributes[:cache_files].map { |value| value.split('#').reverse }]
88
+ files = attributes[:cache_files]
89
+ @cache_files ||= Hash[files.map { |value| interpolate(value).split('#').reverse }]
89
90
  end
90
91
 
91
92
  def env
@@ -1,3 +1,3 @@
1
1
  module Hasta
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.2"
3
3
  end
@@ -4,7 +4,7 @@
4
4
  "onFail": { "ref": "FailureNotify" },
5
5
  "schedule": { "ref": "Nightly" },
6
6
  "runsOn": { "ref": "MenuIntelEMRCluster" },
7
- "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
7
+ "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cacheFile,s3://data-bucket/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/timestamped.yml#timestamped.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
8
8
  "input": [ { "ref": "S3Input1" }, { "ref": "S3Input2" }, { "ref": "S3Input3" } ],
9
9
  "output": { "ref": "S3Output" }
10
10
  }
@@ -19,6 +19,7 @@ describe Hasta::EmrNode do
19
19
  {
20
20
  'mappings.yml' => 's3://data-bucket/path/to/mappings.yml',
21
21
  'ignored.yml' => 's3://data-bucket/path/to/ignored.yml',
22
+ 'timestamped.yml' => 's3://data-bucket/2014-03-28_195039/timestamped.yml'
22
23
  }
23
24
  }
24
25
 
@@ -7,8 +7,18 @@ require 'hasta/local_file_path'
7
7
  describe Hasta::LocalStorage do
8
8
  subject { described_class.new(fog_storage) }
9
9
 
10
+ before do
11
+ Hasta.configure do |conf|
12
+ @original_storage_root = conf.local_storage_root
13
+ conf.local_storage_root = tmpdir
14
+ end
15
+ end
16
+
10
17
  after do
11
18
  FileUtils.rm_rf(tmpdir)
19
+ Hasta.configure do |conf|
20
+ conf.local_storage_root = @original_storage_root
21
+ end
12
22
  end
13
23
 
14
24
  let(:fog_storage) {
@@ -30,7 +40,7 @@ describe Hasta::LocalStorage do
30
40
  let(:content) { "Hi\nBye\nWhy?\n" }
31
41
  let(:data_source) { StringIO.new(content) }
32
42
 
33
- let(:result) { subject.write(s3_uri, data_source) }
43
+ let!(:result) { subject.write(s3_uri, data_source) }
34
44
  let(:local_file_path) { Hasta::LocalFilePath.for(result) }
35
45
 
36
46
  context 'given a directory uri' do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hasta
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - danhodge
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-11 00:00:00.000000000 Z
11
+ date: 2014-08-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fog
@@ -98,16 +98,16 @@ dependencies:
98
98
  name: rspec
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - '>='
101
+ - - ~>
102
102
  - !ruby/object:Gem::Version
103
- version: '0'
103
+ version: 2.13.0
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - '>='
108
+ - - ~>
109
109
  - !ruby/object:Gem::Version
110
- version: '0'
110
+ version: 2.13.0
111
111
  description: Harness for locally testing streaming Hadoop jobs written in Ruby
112
112
  email:
113
113
  - dan@swipely.com