pipely 0.4.4 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c86ce9ade0efc39bc87e6d01085f2dbebd8d4390
-  data.tar.gz: 5c8b4bbcc6381da109a134dde715dd747813c3f2
+  metadata.gz: f66405d01a1c3f031d6f08b65879f011a219d0cb
+  data.tar.gz: 0403bab9c88a0ea033b69c93418932026ef6b177
 SHA512:
-  metadata.gz: 79ebd0a5df8fc75894a1a946a3084326b524487ec8bfbf2fb19db131335aef40a4b19a2d1f08f75bd1967429c581b5ac2506b32e30e5d0dec80acdaa4bb83a7e
-  data.tar.gz: e4d05d61888f1a1196acc18f74c8f0168c641f887710572f505502742fdcb1e03c53abe176bbcca81129f7a38506dda15544f9b9a1bafaad0637e72d57ea5fa6
+  metadata.gz: 9efa8a80f4185ab7cbd7768a585164366fe0c412667e9564df8c821d7660bc53efb8bb711c69c08aec1b3ebab2308e81b57397ba883e330fa38f7c4b903faf8f
+  data.tar.gz: 78415fb04a95d9b6858651c9594fc882614190fb29364c417a7e58b9a776f0b372d3383bfe4cea37b527b66bae8c8f1269fa43735125869d9ebc527837ba7a0e

data/lib/pipely/build/s3_path_builder.rb

@@ -8,6 +8,7 @@ module Pipely
       attr_reader :assets_bucket, :logs_bucket, :steps_bucket
 
       START_TIME = "\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}"
+      START_DATE = "\#{format(@scheduledStartTime,'YYYY-MM-dd')}"
 
       def initialize(options)
         @assets_bucket = options[:assets]
@@ -36,6 +37,10 @@ module Pipely
         "s3n://#{@assets_bucket}/#{@s3prefix}/#{START_TIME}"
       end
 
+      def s3_shared_asset_prefix
+        "s3://#{@assets_bucket}/#{@s3prefix}/shared/#{START_DATE}"
+      end
+
       def bucket_relative_s3_asset_prefix
         "#{@s3prefix}/#{START_TIME}"
       end
@@ -47,6 +52,7 @@ module Pipely
           :s3n_step_prefix => s3n_step_prefix,
           :s3_asset_prefix => s3_asset_prefix,
           :s3n_asset_prefix => s3n_asset_prefix,
+          :s3_shared_asset_prefix => s3_shared_asset_prefix,
           :bucket_relative_s3_asset_prefix => bucket_relative_s3_asset_prefix,
         }
       end

data/lib/pipely/build/template_helpers.rb

@@ -20,6 +20,14 @@ module Pipely
       def streaming_hadoop_step(options)
         parts = [ '/home/hadoop/contrib/streaming/hadoop-streaming.jar' ]
 
+        if jars = options[:lib_jars]
+          parts += Array(jars).map { |jar| ['-libjars', "#{jar}"] }.flatten
+        end
+
+        (options[:defs] || {}).each do |name, value|
+          parts += ['-D', "#{name}=#{value}"]
+        end
+
         Array(options[:input]).each do |input|
           parts += [ '-input', s3n_asset_path(input) ]
         end
@@ -28,6 +36,10 @@ module Pipely
           parts += ['-output', s3_asset_path(output) ]
         end
 
+        if options[:outputformat]
+          parts += ['-outputformat', options[:outputformat] ]
+        end
+
         Array(options[:mapper]).each do |mapper|
           parts += ['-mapper', s3n_step_path(mapper) ]
         end

data/lib/pipely/deploy/client.rb

@@ -18,13 +18,23 @@ module Pipely
         @data_pipelines = Fog::AWS::DataPipeline.new
       end
 
-      def deploy_pipeline(pipeline_name, definition)
+      def deploy_pipeline(pipeline_basename, definition)
+        pipeline_name = [
+          ('P' if ENV['env'] == 'production'),
+          ENV['USER'],
+          pipeline_basename
+        ].compact.join(':')
+
+        tags = { "basename" => pipeline_basename }
+
         # Get a list of all existing pipelines
         pipeline_ids = existing_pipelines(pipeline_name)
         @log.info("#{pipeline_ids.count} existing pipelines: #{pipeline_ids}")
 
         # Create new pipeline
-        created_pipeline_id = create_pipeline(pipeline_name, definition)
+        created_pipeline_id = create_pipeline(pipeline_name,
+                                              definition,
+                                              tags)
         @log.info("Created pipeline id '#{created_pipeline_id}'")
 
         # Delete old pipelines
@@ -54,14 +64,15 @@ module Pipely
         ids
       end
 
-      def create_pipeline(pipeline_name, definition)
+      def create_pipeline(pipeline_name, definition, tags={})
         definition_objects = JSON.parse(definition)['objects']
 
         unique_id = UUIDTools::UUID.random_create
 
         created_pipeline = @data_pipelines.pipelines.create(
           unique_id: unique_id,
-          name: pipeline_name
+          name: pipeline_name,
+          tags: default_tags.merge(tags)
         )
 
         created_pipeline.put(definition_objects)
@@ -74,6 +85,15 @@ module Pipely
         @data_pipelines.pipelines.get(pipeline_id).destroy
       end
 
+    private
+
+      def default_tags
+        {
+          "environment" => ENV['env'],
+          "creator" => ENV['USER']
+        }
+      end
+
     end
   end
 end

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.4.4" unless defined?(::Pipely::VERSION)
+  VERSION = "0.4.8" unless defined?(::Pipely::VERSION)
 end

data/spec/lib/pipely/build/s3_path_builder_spec.rb

@@ -31,6 +31,10 @@ describe Pipely::Build::S3PathBuilder do
     should eq("s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}")
   }
 
+  its(:s3_shared_asset_prefix) {
+    should eq("s3://asset-bucket/run-prefix/shared/\#{format(@scheduledStartTime,'YYYY-MM-dd')}")
+  }
+
   describe "#to_hash" do
     it 'includes the necessary keys for supplying config to a Template' do
       expect(subject.to_hash.keys).to match_array([
@@ -39,6 +43,7 @@ describe Pipely::Build::S3PathBuilder do
         :s3n_step_prefix,
         :s3_asset_prefix,
         :s3n_asset_prefix,
+        :s3_shared_asset_prefix,
         :bucket_relative_s3_asset_prefix,
       ])
     end

data/spec/lib/pipely/build/template_spec.rb

@@ -68,6 +68,20 @@ describe Pipely::Build::Template do
       end
     end
 
+    context "given an outputformat" do
+      it 'points to the outputformat class (not as an S3 URL)' do
+        step = subject.streaming_hadoop_step(
+          :input => '/input_dir/',
+          :output => '/output_dir/',
+          :outputformat => 'com.swipely.foo.outputformat',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer'
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-outputformat,com.swipely.foo.outputformat,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
+      end
+    end
+
     context "given the IdentityReducer" do
       it 'points to the IdentityReducer correctly (not as an S3 URL)' do
         step = subject.streaming_hadoop_step(
@@ -80,6 +94,34 @@ describe Pipely::Build::Template do
         expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
       end
     end
+
+    context "given jar files" do
+      it 'loads the file correctly' do
+        step = subject.streaming_hadoop_step(
+          :input => '/input_dir/',
+          :output => '/output_dir/',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer',
+          :lib_jars => [ 'filter.jar', 'filter2.jar' ],
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-libjars,filter.jar,-libjars,filter2.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
+      end
+    end
+
+    context "given variables" do
+      it 'defines them correctly' do
+        step = subject.streaming_hadoop_step(
+          :input => '/input_dir/',
+          :output => '/output_dir/',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer',
+          :defs => {'name' => 'value'}
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-D,name=value,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
+      end
+    end
   end
 
 end

data/spec/lib/pipely/deploy/client_spec.rb

@@ -6,7 +6,7 @@ describe Pipely::Deploy::Client do
   describe "#deploy_pipeline" do
     let(:existing_pipeline_ids) { ["pipeline-one", "pipeline-two"] }
     let(:new_pipeline_id) { "pipeline-three" }
-    let(:pipeline_name) { "MyPipeline" }
+    let(:pipeline_basename) { "MyPipeline" }
     let(:definition) { "pipeline json" }
 
     it "gets a list of pipelines, creates a new one, and deletes the others" do
@@ -14,14 +14,17 @@ describe Pipely::Deploy::Client do
         and_return(existing_pipeline_ids)
 
       subject.should_receive(:create_pipeline).
-        with(pipeline_name, anything()).
+        with("#{ENV['USER']}:#{pipeline_basename}",
+             anything(),
+             hash_including( 'basename' => pipeline_basename )
+        ).
         and_return(new_pipeline_id)
 
       existing_pipeline_ids.each do |id|
         subject.should_receive(:delete_pipeline).with(id)
       end
 
-      subject.deploy_pipeline(pipeline_name, definition)
+      subject.deploy_pipeline(pipeline_basename, definition)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.8
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-03 00:00:00.000000000 Z
+date: 2014-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.21.0
+        version: 1.22.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.21.0
+        version: 1.22.0
 - !ruby/object:Gem::Dependency
   name: unf
   requirement: !ruby/object:Gem::Requirement