hasta 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 181a02eb6d37332af6efde332af20305f692f3e0
-  data.tar.gz: b484f8ad5a8884b7f13e03caee7277f6b00d521b
+  metadata.gz: 5dbce97906a4409f9621114228fd88a9d76ef11f
+  data.tar.gz: 077cf2b8bad29d39b688a02ba71fae387f4ca97e
 SHA512:
-  metadata.gz: a2206da5b8b905a66037ac9c23e7faab4c4a3254ef127988620bbe36eb6a04273677d8cf42785dacb889c0b7b35c9636549462a80ca6849f2b576839141b9756
-  data.tar.gz: c7d96b9a9d4383e25156cad5ef58250b9713881eb70251b83347724264f08ef7adbe8ecdbc261f17c776bb12d6106466beb167a1284736fd8a0147a79d2bef25
+  metadata.gz: 044dc81cd6ff504a15a00b51745de17a58b900f843fc8931af8572f4fe67eb770a3cb7a00ff03d8b7b749c56ef4bd79a4e38609ef14d10685ea604be9ef8502f
+  data.tar.gz: f60dd1d7cdb3b4ef275294c97cd91d4c7ec9a227f3c3132ab22b6bb647239f70da49f849a3ecf2f340caec319b1e549c305d51d112f321b6b255ddddb3cfa5f1

data/README.md

@@ -34,7 +34,8 @@ Or install it yourself as:
   Hasta::Tasks::Runner.new do |task, opts|
     task.definition_file = <path-to-AWS-datapipeline-definition-json-file>
     task.job_id = opts[:job_id]
-    task.scheduled_start_time = Time.parse(opts[:scheduled_start_time])
+    scheduled_start_time = opts[:scheduled_start_time]
+    task.scheduled_start_time = scheduled_start_time.nil? ? Time.now : Time.parse(scheduled_start_time)
     task.project_root = File.dirname(__FILE__)
   end
 ```
@@ -53,6 +54,7 @@ Or install it yourself as:
 ## Configuration
 
 The following code snippet illustrates how to update the global Hasta configuration, which values are mandatory, and which values have defaults.
+This should go in the Rakefile as well.
 
 ```ruby
 Hasta.configure do |config|

data/hasta.gemspec

@@ -25,5 +25,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "cane"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec", '~> 2.13.0'
 end

data/lib/hasta/emr_node.rb

@@ -85,7 +85,8 @@ module Hasta
     end
 
     def cache_files
-      @cache_files ||= Hash[attributes[:cache_files].map { |value| value.split('#').reverse }]
+      files = attributes[:cache_files]
+      @cache_files ||= Hash[files.map { |value| interpolate(value).split('#').reverse }]
     end
 
     def env

data/lib/hasta/version.rb

@@ -1,3 +1,3 @@
 module Hasta
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/spec/fixtures/hasta/json/emr_node.json

@@ -4,7 +4,7 @@
   "onFail": { "ref": "FailureNotify" },
   "schedule": { "ref": "Nightly" },
   "runsOn": { "ref": "MenuIntelEMRCluster" },
-  "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
+  "step": "/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input1/,-output,s3://data-bucket/path/to/data/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output/,-mapper,cat,-reducer,s3n://steps-bucket/path/to/reducer.rb,-cacheFile,s3://data-bucket/path/to/mappings.yml#mappings.yml,-cacheFile,s3://data-bucket/path/to/ignored.yml#ignored.yml,-cacheFile,s3://data-bucket/#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/timestamped.yml#timestamped.yml,-cmdenv,API_KEY=123456,-cmdenv,ENVIRONMENT_NAME=uat",
   "input": [ { "ref": "S3Input1" }, { "ref": "S3Input2" }, { "ref": "S3Input3" } ],
   "output": { "ref": "S3Output" }
 }

data/spec/hasta/emr_node_spec.rb

@@ -19,6 +19,7 @@ describe Hasta::EmrNode do
     {
       'mappings.yml' => 's3://data-bucket/path/to/mappings.yml',
       'ignored.yml' => 's3://data-bucket/path/to/ignored.yml',
+      'timestamped.yml' => 's3://data-bucket/2014-03-28_195039/timestamped.yml'
     }
   }
 

data/spec/hasta/local_storage_spec.rb

@@ -7,8 +7,18 @@ require 'hasta/local_file_path'
 describe Hasta::LocalStorage do
   subject { described_class.new(fog_storage) }
 
+  before do
+    Hasta.configure do |conf|
+      @original_storage_root = conf.local_storage_root
+      conf.local_storage_root = tmpdir
+    end
+  end
+
   after do
     FileUtils.rm_rf(tmpdir)
+    Hasta.configure do |conf|
+      conf.local_storage_root = @original_storage_root
+    end
   end
 
   let(:fog_storage) {
@@ -30,7 +40,7 @@ describe Hasta::LocalStorage do
     let(:content) { "Hi\nBye\nWhy?\n" }
     let(:data_source) { StringIO.new(content) }
 
-    let(:result) { subject.write(s3_uri, data_source) }
+    let!(:result) { subject.write(s3_uri, data_source) }
     let(:local_file_path) { Hasta::LocalFilePath.for(result) }
 
     context 'given a directory uri' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hasta
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - danhodge
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-11 00:00:00.000000000 Z
+date: 2014-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -98,16 +98,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.13.0
 description: Harness for locally testing streaming Hadoop jobs written in Ruby
 email:
 - dan@swipely.com