pipely 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96b0105ea802c8495325b0b01df591a8a95369dc
-  data.tar.gz: 02ea6bc3048878f4fd9fed396837179b6cc0302e
+  metadata.gz: e753a1c8b9207a58a13e7f4454b4d777ee2b42ba
+  data.tar.gz: af3d1bbe539a028bcbc58c5e6f1ca2df908f6a41
 SHA512:
-  metadata.gz: 1393a450e4c71d8d9dc5477978fac3546092997f6112608c8bafb04dbc65efd5061f04e5eb232c70a816f05ff88a3e6615613afbf35657e459d44e60a0dc7fe7
-  data.tar.gz: 29cfcc8c63636d99a2de3ccedfd6574f7a67784be90f4d731606a6674477c25c313b8d7e30a72e9019012700af55c5c8de3962eefa568560981d8b75cec070ec
+  metadata.gz: 56fdf912f37ebb81a7f17a7a8a6c5686228165045d5917a48f12621b4a83357b7bb73fcb0e3562c0a0df9f53ec8d5131d484056d0f8283a819b1f04daf0c46f9
+  data.tar.gz: 6442124a52915c429e9a2e0ada617b687c10c5db887b2c84f1b77f3a96ec1e4e7a0cd789240e594f82807ab5821ebf1f973960654a35a7f29c7375cc4d2d1708

data/lib/pipely/build.rb

@@ -0,0 +1,31 @@
+require 'pipely/build/definition'
+require 'pipely/build/template'
+require 'pipely/build/daily_scheduler'
+require 'pipely/build/right_now_scheduler'
+require 'pipely/build/s3_path_builder'
+require 'pipely/build/environment_config'
+
+module Pipely
+
+  # Turn Templates+config into a deployable Definition.
+  #
+  module Build
+
+    def self.build_definition(template, environment, config_path)
+      env = environment.to_sym
+      config = EnvironmentConfig.load(config_path, env)
+
+      case environment.to_sym
+      when :production
+        s3_prefix = "production/#{config[:namespace]}"
+        scheduler = DailyScheduler.new
+      when :staging
+        s3_prefix = "staging/#{`whoami`.strip}/#{config[:namespace]}"
+        scheduler = RightNowScheduler.new
+      end
+
+      Definition.new(template, env, s3_prefix, scheduler, config)
+    end
+
+  end
+end

data/lib/pipely/build/daily_scheduler.rb

@@ -0,0 +1,37 @@
+module Pipely
+  module Build
+
+    # Compute schedule attributes for a pipeline that runs once-a-day at a set
+    # time.
+    #
+    class DailyScheduler
+
+      def initialize(start_time="11:00:00")
+        @start_time = start_time
+      end
+
+      def period
+        '24 hours'
+      end
+
+      def start_date_time
+        date = Date.today
+
+        # if start_time already happened today, wait for tomorrow's start_time
+        now_time = Time.now.utc.strftime('%H:%M:%S')
+        date += 1 if now_time >= @start_time
+
+        date.strftime("%Y-%m-%dT#{@start_time}")
+      end
+
+      def to_hash
+        {
+          :period => period,
+          :start_date_time => start_date_time
+        }
+      end
+
+    end
+
+  end
+end

data/lib/pipely/build/definition.rb

@@ -0,0 +1,26 @@
+module Pipely
+  module Build
+
+    # Represent a pipeline definition, built from a Template and some config.
+    #
+    class Definition < Struct.new(:template,:env,:s3_prefix,:scheduler,:config)
+      def pipeline_name
+        config[:name]
+      end
+
+      def s3_path_builder
+        S3PathBuilder.new(config[:s3].merge(prefix: s3_prefix))
+      end
+
+      def to_json
+        template.apply_config(:environment => env)
+        template.apply_config(config)
+        template.apply_config(s3_path_builder.to_hash)
+        template.apply_config(scheduler.to_hash)
+
+        template.to_json
+      end
+    end
+
+  end
+end

data/lib/pipely/build/environment_config.rb

@@ -0,0 +1,34 @@
+require 'yaml'
+
+module Pipely
+  module Build
+
+    # Work with YAML config files that contain parallel configs for various
+    # environments.
+    #
+    class EnvironmentConfig < Hash
+
+      def self.load(filename, environment)
+        raw = YAML.load_file(filename)[environment.to_s]
+        load_from_hash(raw)
+      end
+
+      def self.load_from_hash(attributes)
+        config = new
+
+        attributes.each do |k, v|
+          case v
+          when Hash
+            config[k.to_sym] = load_from_hash(v)
+          else
+            config[k.to_sym] = v.clone
+          end
+        end
+
+        config
+      end
+
+    end
+
+  end
+end

data/lib/pipely/build/right_now_scheduler.rb

@@ -0,0 +1,27 @@
+module Pipely
+  module Build
+
+    # Compute schedule attributes for a pipeline that should run immediately
+    # after being deployed.
+    #
+    class RightNowScheduler
+
+      def period
+        '12 hours'
+      end
+
+      def start_date_time
+        Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S")
+      end
+
+      def to_hash
+        {
+          :period => period,
+          :start_date_time => start_date_time
+        }
+      end
+
+    end
+
+  end
+end

data/lib/pipely/build/s3_path_builder.rb

@@ -0,0 +1,52 @@
+module Pipely
+  module Build
+
+    # Builds paths to assets, logs, and steps that are on S3.
+    #
+    class S3PathBuilder
+
+      attr_reader :assets_bucket, :logs_bucket, :steps_bucket
+
+      START_TIME = "\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}"
+
+      def initialize(options)
+        @assets_bucket = options[:assets]
+        @logs_bucket = options[:logs]
+        @steps_bucket = options[:steps]
+        @s3prefix = options[:prefix]
+      end
+
+      def s3_log_prefix
+        "s3://#{@logs_bucket}/#{@s3prefix}/#{START_TIME}"
+      end
+
+      def s3_step_prefix
+        "s3://#{@steps_bucket}/#{@s3prefix}"
+      end
+
+      def s3n_step_prefix
+        "s3n://#{@steps_bucket}/#{@s3prefix}"
+      end
+
+      def s3_asset_prefix
+        "s3://#{@assets_bucket}/#{@s3prefix}/#{START_TIME}"
+      end
+
+      def s3n_asset_prefix
+        "s3n://#{@assets_bucket}/#{@s3prefix}/#{START_TIME}"
+      end
+
+      def to_hash
+        {
+          :s3_log_prefix => s3_log_prefix,
+          :s3_step_prefix => s3_step_prefix,
+          :s3n_step_prefix => s3n_step_prefix,
+          :s3_asset_prefix => s3_asset_prefix,
+          :s3n_asset_prefix => s3n_asset_prefix
+        }
+      end
+
+    end
+
+  end
+end

data/lib/pipely/build/template.rb

@@ -0,0 +1,43 @@
+require 'active_support/core_ext/hash'
+require 'erb'
+
+require 'pipely/build/template_helpers'
+
+module Pipely
+  module Build
+
+    # An ERB template that can be interpolated with config hashes to render a
+    # deployable pipeline definition.
+    #
+    class Template
+      include TemplateHelpers
+
+      def initialize(source)
+        @source = source
+        @config = {}
+      end
+
+      def apply_config(attributes)
+        @config.merge!(attributes.symbolize_keys)
+      end
+
+      def to_json
+        ERB.new(@source).result(binding)
+      end
+
+      def respond_to_missing(method_name, include_private=false)
+        @config.keys.include?(method_name.to_s) || super
+      end
+
+      def method_missing(method_name, *args, &block)
+        if @config.keys.include?(method_name)
+          @config[method_name]
+        else
+          super
+        end
+      end
+
+    end
+
+  end
+end

data/lib/pipely/build/template_helpers.rb

@@ -0,0 +1,53 @@
+module Pipely
+  module Build
+
+    # Helper methods used by ERB templates.
+    #
+    module TemplateHelpers
+
+      def s3_asset_path(path)
+        "#{s3_asset_prefix if '/' == path[0]}#{path}"
+      end
+
+      def s3n_asset_path(path)
+        "#{s3n_asset_prefix if '/' == path[0]}#{path}"
+      end
+
+      def s3n_step_path(path)
+        "#{s3n_step_prefix if '/' == path[0]}#{path}"
+      end
+
+      def streaming_hadoop_step(options)
+        parts = [ '/home/hadoop/contrib/streaming/hadoop-streaming.jar' ]
+
+        Array(options[:input]).each do |input|
+          parts += [ '-input', s3n_asset_path(input) ]
+        end
+
+        Array(options[:output]).each do |output|
+          parts += ['-output', s3_asset_path(output) ]
+        end
+
+        Array(options[:mapper]).each do |mapper|
+          parts += ['-mapper', s3n_step_path(mapper) ]
+        end
+
+        Array(options[:reducer]).each do |reducer|
+          parts += ['-reducer', s3n_step_path(reducer) ]
+        end
+
+        Array(options[:cache_file]).each do |cache_file|
+          parts += ['-cacheFile', s3n_asset_path(cache_file)]
+        end
+
+        (options[:env] || {}).each do |name, value|
+          parts += ['-cmdenv', "#{name}=#{value}"]
+        end
+
+        parts.join(',')
+      end
+
+    end
+
+  end
+end

data/lib/pipely/deploy.rb

@@ -0,0 +1 @@
+require 'pipely/deploy/client'

data/lib/pipely/deploy/client.rb

@@ -0,0 +1,79 @@
+require 'fog'
+require 'logger'
+require 'tempfile'
+require 'uuidtools'
+
+module Pipely
+  module Deploy
+
+    # Client for managing deployment of rendered definitions.
+    #
+    class Client
+
+      # Generic error representing failure to deploy a rendered definition.
+      class PipelineDeployerError < RuntimeError; end
+
+      def initialize(log=nil)
+        @log = log || Logger.new(STDOUT)
+        @data_pipelines = Fog::AWS::DataPipeline.new
+      end
+
+      def deploy_pipeline(pipeline_name, definition)
+        # Get a list of all existing pipelines
+        pipeline_ids = existing_pipelines(pipeline_name)
+        @log.info("#{pipeline_ids.count} existing pipelines: #{pipeline_ids}")
+
+        # Create new pipeline
+        created_pipeline_id = create_pipeline(pipeline_name, definition)
+        @log.info("Created pipeline id '#{created_pipeline_id}'")
+
+        # Delete old pipelines
+        pipeline_ids.each do |pipeline_id|
+          begin
+            delete_pipeline(pipeline_id)
+            @log.info("Deleted pipeline '#{pipeline_id}'")
+
+          rescue PipelineDeployerError => error
+            @log.warn(error)
+          end
+        end
+      end
+
+      def existing_pipelines(pipeline_name)
+        ids = []
+
+        begin
+          result = Fog::AWS[:data_pipeline].list_pipelines
+
+          ids += result['pipelineIdList'].
+                   select { |p| p['name'] == pipeline_name }.
+                   map { |p| p['id'] }
+
+        end while (result['hasMoreResults'] && result['marker'])
+
+        ids
+      end
+
+      def create_pipeline(pipeline_name, definition)
+        definition_objects = JSON.parse(definition)['objects']
+
+        unique_id = UUIDTools::UUID.random_create
+
+        created_pipeline = @data_pipelines.pipelines.create(
+          unique_id: unique_id,
+          name: pipeline_name
+        )
+
+        created_pipeline.put(definition_objects)
+        created_pipeline.activate
+
+        created_pipeline.id
+      end
+
+      def delete_pipeline(pipeline_id)
+        @data_pipelines.pipelines.get(pipeline_id).destroy
+      end
+
+    end
+  end
+end

data/lib/pipely/tasks.rb

@@ -0,0 +1,9 @@
+require 'pipely/tasks/upload_steps'
+require 'pipely/tasks/deploy'
+require 'pipely/tasks/graph'
+require 'pipely/tasks/definition'
+
+module Pipely
+  module Tasks
+  end
+end

data/lib/pipely/tasks/definition.rb

@@ -0,0 +1,69 @@
+require 'rake'
+require 'rake/tasklib'
+require 'pipely'
+
+module Pipely
+  module Tasks
+    class Definition < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of task.
+      #
+      # default:
+      #   :definition
+      attr_accessor :name
+
+      # Path where rendered definitions are written.
+      #
+      # default:
+      #   "definitions"
+      attr_accessor :path
+
+      # Pipeline definition instance
+      attr_accessor :definition
+
+      # Use verbose output. If this is set to true, the task will print the
+      # local and remote paths of each step file it uploads to S3.
+      #
+      # default:
+      #   true
+      attr_accessor :verbose
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        directory path
+
+        desc "Graphs the full pipeline definition using Graphviz"
+        task name => path do |_, task_args|
+          RakeFileUtils.send(:verbose, verbose) do
+            if task_block
+              task_block.call(*[self, task_args].slice(0, task_block.arity))
+            end
+
+            run_task verbose
+          end
+        end
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || :definition
+        @verbose = true
+        @path = "definitions"
+      end
+
+      def run_task(verbose)
+        puts "Generating #{target_filename}" if verbose
+
+        File.open(target_filename, 'w') do |file|
+          file.write(definition.to_json)
+        end
+      end
+
+      def target_filename
+        "#{path}/#{definition.base_filename}.json"
+      end
+
+    end
+  end
+end

data/lib/pipely/tasks/deploy.rb

@@ -0,0 +1,58 @@
+require 'rake'
+require 'rake/tasklib'
+require 'pipely/deploy'
+
+module Pipely
+  module Tasks
+    class Deploy < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of task.
+      #
+      # default:
+      #   :deploy
+      attr_accessor :name
+
+      # Pipeline definition instance
+      attr_accessor :definition
+
+      # Use verbose output. If this is set to true, the task will print the
+      # local and remote paths of each step file it uploads to S3.
+      #
+      # default:
+      #   true
+      attr_accessor :verbose
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        desc "Deploy pipeline" unless ::Rake.application.last_comment
+
+        task name, *args do |_, task_args|
+          RakeFileUtils.send(:verbose, verbose) do
+            if task_block
+              task_block.call(*[self, task_args].slice(0, task_block.arity))
+            end
+
+            run_task verbose
+          end
+        end
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || :deploy
+        @verbose = true
+      end
+
+      def run_task(verbose)
+        Rake::Task["upload_steps"].invoke
+
+        Pipely::Deploy::Client.new.deploy_pipeline(
+          definition.pipeline_name,
+          definition.to_json
+        )
+      end
+
+    end
+  end
+end

data/lib/pipely/tasks/graph.rb

@@ -0,0 +1,75 @@
+require 'rake'
+require 'rake/tasklib'
+require 'pipely'
+
+module Pipely
+  module Tasks
+    class Graph < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of task.
+      #
+      # default:
+      #   :graph
+      attr_accessor :name
+
+      # Path to write graph images to.
+      #
+      # default:
+      #   "graphs"
+      attr_accessor :path
+
+      # Pipeline definition instance
+      attr_accessor :definition
+
+      # Use verbose output. If this is set to true, the task will print the
+      # local and remote paths of each step file it uploads to S3.
+      #
+      # default:
+      #   true
+      attr_accessor :verbose
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        # create the `path` directory if it doesn't exist
+        directory path
+
+        namespace name do
+          task :full => path do |_, task_args|
+            RakeFileUtils.send(:verbose, verbose) do
+              if task_block
+                task_block.call(*[self, task_args].slice(0, task_block.arity))
+              end
+
+              run_task verbose
+            end
+          end
+
+          task :open => :full do
+            `open #{target_filename}`
+          end
+        end
+
+        desc "Graphs the full pipeline definition using Graphviz"
+        task name => "#{name}:full"
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || :graph
+        @verbose = true
+        @path = "graphs"
+      end
+
+      def run_task(verbose)
+        puts "Generating #{target_filename}" if verbose
+        Pipely.draw(definition.to_json, target_filename)
+      end
+
+      def target_filename
+        "#{path}/#{definition.base_filename}.png"
+      end
+
+    end
+  end
+end

data/lib/pipely/tasks/upload_steps.rb

@@ -0,0 +1,86 @@
+require 'rake'
+require 'rake/tasklib'
+require 'fog'
+
+module Pipely
+  module Tasks
+    class UploadSteps < ::Rake::TaskLib
+      include ::Rake::DSL if defined?(::Rake::DSL)
+
+      # Name of task.
+      #
+      # default:
+      #   :upload_steps
+      attr_accessor :name
+
+      # Local path to where the step files are.
+      #
+      # default:
+      #   "steps"
+      attr_accessor :local_path
+
+      # Name of S3 bucket to upload steps to.
+      attr_accessor :s3_bucket_name
+
+      # Path within S3 bucket to upload steps to.
+      attr_accessor :s3_path
+
+      # Use verbose output. If this is set to true, the task will print the
+      # local and remote paths of each step file it uploads to S3.
+      #
+      # default:
+      #   true
+      attr_accessor :verbose
+
+      def initialize(*args, &task_block)
+        setup_ivars(args)
+
+        unless ::Rake.application.last_comment
+          desc "Upload Data Pipeline steps to S3"
+        end
+
+        task name, *args do |_, task_args|
+          RakeFileUtils.send(:verbose, verbose) do
+            if task_block
+              task_block.call(*[self, task_args].slice(0, task_block.arity))
+            end
+
+            run_task verbose
+          end
+        end
+      end
+
+      def setup_ivars(args)
+        @name = args.shift || :upload_steps
+        @verbose = true
+        @local_path = "steps"
+      end
+
+      def run_task(verbose)
+        with_bucket do |directory|
+          step_files.each do |file_name|
+            dest = "#{s3_path}/#{File.basename(file_name)}"
+            puts "uploading #{dest}" if verbose
+            directory.files.create(key: dest, body: File.read(file_name))
+          end
+        end
+      end
+
+    private
+
+      def with_bucket
+        storage = Fog::Storage.new({ provider: 'AWS' })
+        if directory = storage.directories.detect{ |d| d.key == s3_bucket_name }
+          yield(directory)
+        else
+          raise "Couldn't find S3 bucket '#{s3_bucket_name}'"
+        end
+      end
+
+      def step_files
+        FileList.new(File.join(local_path, "*"))
+      end
+
+    end
+  end
+end

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.1.5" unless defined?(::DataPipelineGraphviz::VERSION)
+  VERSION = "0.2.0" unless defined?(::DataPipelineGraphviz::VERSION)
 end

data/spec/lib/pipely/build/daily_scheduler_spec.rb

@@ -0,0 +1,33 @@
+require 'pipely/build/daily_scheduler'
+
+describe Pipely::Build::DailyScheduler do
+
+  let(:start_time) { "11:00:00" }
+
+  subject { described_class.new(start_time) }
+
+  describe "#period" do
+    it "is '24 hours'" do
+      expect(subject.period).to eq('24 hours')
+    end
+  end
+
+  describe "#start_date_time" do
+    context "if the start time has already happened today in UTC" do
+      it "chooses the start time tomorrow" do
+        Timecop.freeze(Time.utc(2013, 6, 12, 16, 12, 30)) do
+          expect(subject.start_date_time).to eq("2013-06-13T11:00:00")
+        end
+      end
+    end
+
+    context "if the start time has not happened yet today in UTC" do
+      it "chooses the start time today" do
+        Timecop.freeze(Time.utc(2013, 6, 13, 4, 12, 30)) do
+          expect(subject.start_date_time).to eq("2013-06-13T11:00:00")
+        end
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/build/right_now_scheduler_spec.rb

@@ -0,0 +1,19 @@
+require 'pipely/build/right_now_scheduler'
+
+describe Pipely::Build::RightNowScheduler do
+
+  describe "#period" do
+    it "is '12 hours'" do
+      expect(subject.period).to eq('12 hours')
+    end
+  end
+
+  describe "#start_date_time" do
+    it "chooses the current time as the start time" do
+      Timecop.freeze(Time.utc(2013, 6, 12, 16, 12, 30)) do
+        expect(subject.start_date_time).to eq("2013-06-12T16:12:30")
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/build/s3_path_builder_spec.rb

@@ -0,0 +1,46 @@
+require 'pipely/build/s3_path_builder'
+
+describe Pipely::Build::S3PathBuilder do
+
+  subject {
+    described_class.new(
+      logs: 'log-bucket',
+      steps: 'step-bucket',
+      assets: 'asset-bucket',
+      prefix: 'run-prefix',
+    )
+  }
+
+  its(:s3_log_prefix) {
+    should eq("s3://log-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}")
+  }
+
+  its(:s3_step_prefix) {
+    should eq("s3://step-bucket/run-prefix")
+  }
+
+  its(:s3n_step_prefix) {
+    should eq("s3n://step-bucket/run-prefix")
+  }
+
+  its(:s3_asset_prefix) {
+    should eq("s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}")
+  }
+
+  its(:s3n_asset_prefix) {
+    should eq("s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}")
+  }
+
+  describe "#to_hash" do
+    it 'includes the necessary keys for supplying config to a Template' do
+      expect(subject.to_hash.keys).to match_array([
+        :s3_log_prefix,
+        :s3_step_prefix,
+        :s3n_step_prefix,
+        :s3_asset_prefix,
+        :s3n_asset_prefix,
+      ])
+    end
+  end
+
+end

data/spec/lib/pipely/build/template_spec.rb

@@ -0,0 +1,85 @@
+require 'pipely/build/template'
+
+describe Pipely::Build::Template do
+  let(:source) { "some test json <%= foo %>" }
+
+  subject { described_class.new(source) }
+
+  context 'given some configuration' do
+    let(:foo) { 'asdfgwrytqfadfa' }
+    let(:expected_json) { "some test json #{foo}" }
+
+    before do
+      subject.apply_config({ foo: foo })
+    end
+
+    its(:to_json) { should eq(expected_json) }
+  end
+
+  describe "#streaming_hadoop_step(options)" do
+    let(:s3_path_builder) {
+      Pipely::Build::S3PathBuilder.new(
+        logs: 'log-bucket',
+        steps: 'step-bucket',
+        assets: 'asset-bucket',
+        prefix: 'run-prefix'
+      )
+    }
+
+    before do
+      subject.apply_config(s3_path_builder.to_hash)
+    end
+
+    it "builds a streaming hadoop step" do
+      step = subject.streaming_hadoop_step(
+        :input => '/input_dir/',
+        :output => '/output_dir/',
+        :mapper => '/mapper.rb',
+        :reducer => '/reducer.rb'
+      )
+
+      expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,s3n://step-bucket/run-prefix/reducer.rb")
+    end
+
+    context "given an array of inputs" do
+      it 'points to the IdentityReducer correctly (not as an S3 URL)' do
+        step = subject.streaming_hadoop_step(
+          :input => ['/input_dir/', '/input_dir2/'],
+          :output => '/output_dir/',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer'
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir2/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
+      end
+    end
+
+    context "given a cacheFile" do
+      it 'points to the IdentityReducer correctly (not as an S3 URL)' do
+        step = subject.streaming_hadoop_step(
+          :input => '/input_dir/',
+          :output => '/output_dir/',
+          :cache_file => '/cache_file#cache_file',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer'
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer,-cacheFile,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/cache_file#cache_file")
+      end
+    end
+
+    context "given the IdentityReducer" do
+      it 'points to the IdentityReducer correctly (not as an S3 URL)' do
+        step = subject.streaming_hadoop_step(
+          :input => '/input_dir/',
+          :output => '/output_dir/',
+          :mapper => '/mapper.rb',
+          :reducer => 'org.apache.hadoop.mapred.lib.IdentityReducer'
+        )
+
+        expect(step).to eq("/home/hadoop/contrib/streaming/hadoop-streaming.jar,-input,s3n://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/input_dir/,-output,s3://asset-bucket/run-prefix/\#{format(@scheduledStartTime,'YYYY-MM-dd_HHmmss')}/output_dir/,-mapper,s3n://step-bucket/run-prefix/mapper.rb,-reducer,org.apache.hadoop.mapred.lib.IdentityReducer")
+      end
+    end
+  end
+
+end

data/spec/lib/pipely/build_spec.rb

@@ -0,0 +1,3 @@
+describe Pipely::Build do
+
+end

data/spec/lib/pipely/deploy/client_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'pipely/deploy'
+
+describe Pipely::Deploy::Client do
+
+  describe "#deploy_pipeline" do
+    let(:existing_pipeline_ids) { ["pipeline-one", "pipeline-two"] }
+    let(:new_pipeline_id) { "pipeline-three" }
+    let(:pipeline_name) { "MyPipeline" }
+    let(:definition) { "pipeline json" }
+
+    it "gets a list of pipelines, creates a new one, and deletes the others" do
+      subject.should_receive(:existing_pipelines).
+        and_return(existing_pipeline_ids)
+
+      subject.should_receive(:create_pipeline).
+        with(pipeline_name, anything()).
+        and_return(new_pipeline_id)
+
+      existing_pipeline_ids.each do |id|
+        subject.should_receive(:delete_pipeline).with(id)
+      end
+
+      subject.deploy_pipeline(pipeline_name, definition)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'timecop'
+require 'fog'
+
+Fog.credentials = {
+  aws_access_key_id: "xxx",
+  aws_secret_access_key: "xxx"
+}
+
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2013-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.19.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.18.0
+        version: 1.19.0
 - !ruby/object:Gem::Dependency
   name: unf
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,34 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -108,6 +136,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - matt@swipely.com
@@ -116,24 +158,46 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/pipely/build/daily_scheduler.rb
+- lib/pipely/build/definition.rb
+- lib/pipely/build/environment_config.rb
+- lib/pipely/build/right_now_scheduler.rb
+- lib/pipely/build/s3_path_builder.rb
+- lib/pipely/build/template.rb
+- lib/pipely/build/template_helpers.rb
+- lib/pipely/build.rb
 - lib/pipely/component.rb
 - lib/pipely/definition.rb
 - lib/pipely/dependency.rb
+- lib/pipely/deploy/client.rb
+- lib/pipely/deploy.rb
 - lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
 - lib/pipely/live_pipeline.rb
 - lib/pipely/reference_list.rb
 - lib/pipely/runs_report.rb
+- lib/pipely/tasks/definition.rb
+- lib/pipely/tasks/deploy.rb
+- lib/pipely/tasks/graph.rb
+- lib/pipely/tasks/upload_steps.rb
+- lib/pipely/tasks.rb
 - lib/pipely/version.rb
 - lib/pipely.rb
 - Rakefile
 - README.md
+- spec/lib/pipely/build/daily_scheduler_spec.rb
+- spec/lib/pipely/build/right_now_scheduler_spec.rb
+- spec/lib/pipely/build/s3_path_builder_spec.rb
+- spec/lib/pipely/build/template_spec.rb
+- spec/lib/pipely/build_spec.rb
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/client_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
 - spec/lib/pipely_spec.rb
+- spec/spec_helper.rb
 - bin/pipely
 homepage: http://github.com/swipely/pipely
 licenses:
@@ -160,9 +224,16 @@ signing_key:
 specification_version: 4
 summary: Generate dependency graphs from pipeline definitions.
 test_files:
+- spec/lib/pipely/build/daily_scheduler_spec.rb
+- spec/lib/pipely/build/right_now_scheduler_spec.rb
+- spec/lib/pipely/build/s3_path_builder_spec.rb
+- spec/lib/pipely/build/template_spec.rb
+- spec/lib/pipely/build_spec.rb
 - spec/lib/pipely/component_spec.rb
 - spec/lib/pipely/definition_spec.rb
 - spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/deploy/client_spec.rb
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
 - spec/lib/pipely_spec.rb
+- spec/spec_helper.rb