pipely 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41fc3328b9562c634d8431f24de10627f682e827
-  data.tar.gz: 7bdc7a4693e34194baa787be589e1ca4e28d561d
+  metadata.gz: 9b9b109c0262cd83b9db819b8464e1ed39a6bef6
+  data.tar.gz: 633b3e1c8bb00323199629919714dec0cce1523f
 SHA512:
-  metadata.gz: c6bb2fb0e3872e128241b0760ef95ddc97a5c7baf534439eae8dcfaa76d7d115196bb1254e98723125dbadf5c5aba485c7d65ce183bb1e593149d2a2a23c7bca
-  data.tar.gz: 13495dbb6503f46cac2cd52682f9b3245e38970641e69081d59e13922f37987d3ac169253cf9fc68601564883bbee68523cfb0d4677386ef15720f88e7c8feb0
+  metadata.gz: 9576d0aeb1273ebf96021d60cd1f6fb98f9f800cdae3bcd14a1ea6ca5919864ba3896ff99668c340ef0f25b15fe1280997ca7f997ccabe78ddbb4709762c216a
+  data.tar.gz: ef09a07d9ecdb00ea13633455b88f262336756e3e8d51889dd989be384b0ce01192ccbc7c9d474b21d436a2a055c67c1d134bf9ed83786af47add4571b5a2a12

data/bin/pipely

@@ -1,40 +1,16 @@
 #!/usr/bin/env ruby
 
 require 'pipely'
-require 'optparse'
 
-pipeline_id = input_path = output_path = nil
-verbose = automatic_open = false
+options = Pipely::Options.parse
 
-OptionParser.new do |opts|
-  opts.banner = "Usage: pipely [options]"
+if options.pipeline_id
+  Pipely::Actions::GraphLivePipeline.new(options).execute
 
-  opts.on("-p", "--pipeline-id PIPELINE_ID",
-          "ID of a live pipeline to visualize with live execution statuses") do |id|
-    pipeline_id = id
-  end
+elsif options.input_path
+  Pipely::Actions::GraphFilePipeline.new(options).execute
 
-  opts.on("-i", "--input PATH",
-          "Path to a JSON pipeline definition file to visualize") do |input|
-    input_path = input
-  end
+else
+  Pipely::Actions::ListLivePipelines.new(options).execute
 
-  opts.on("-o", "--output PATH",
-          "Path to write PNG file(s) of graph visualization(s)") do |output|
-    output_path = output
-  end
-end.parse!
-
-if pipeline_id
-  live_pipeline = Pipely::LivePipeline.new(pipeline_id)
-  live_pipeline.print_runs_report
-  live_pipeline.render_graphs
-
-elsif input_path
-  definition_json = File.open(input_path).read
-  output_base = File.basename(input_path,".*") + '.png'
-  output_file = output_path ? File.join(output_path, output_base) : output_base
-
-  puts "Generating #{output_file}"
-  Pipely.draw(definition_json, output_file)
 end

data/lib/pipely.rb

@@ -1,6 +1,9 @@
+require 'pipely/options'
+require 'pipely/actions'
 require 'pipely/definition'
 require 'pipely/graph_builder'
 require 'pipely/live_pipeline'
+require 'pipely/s3_writer'
 
 # The top-level module for this gem. It provides the recommended public
 # interface for using Pipely to visualize and manipulate your Data Pipeline
@@ -18,7 +21,14 @@ module Pipely
     graph_builder = GraphBuilder.new
 
     graph = graph_builder.build(definition.components_for_graph)
-    graph.output( :png => filename )
+
+    if filename.start_with?('s3://')
+      content = graph.output( :png => String )
+      S3Writer.new(filename).write(content)
+    else
+      graph.output( :png => filename )
+      filename
+    end
   end
 
 end

data/lib/pipely/actions.rb

@@ -0,0 +1,8 @@
+require 'pipely/actions/graph_live_pipeline'
+require 'pipely/actions/graph_file_pipeline'
+require 'pipely/actions/list_live_pipelines'
+
+module Pipely
+  module Actions
+  end
+end

data/lib/pipely/actions/graph_file_pipeline.rb

@@ -0,0 +1,39 @@
+module Pipely
+  module Actions
+
+    # Graph a pipeline definition from a file.
+    #
+    class GraphFilePipeline
+
+      def initialize(options)
+        @options = options
+      end
+
+      def execute
+        puts "Generating #{output_file}"
+        Pipely.draw(definition_json, output_file)
+      end
+
+    private
+
+      def definition_json
+        File.open(@options.input_path).read
+      end
+
+      def output_base
+        @output_base ||= File.basename(@options.input_path,".*") + '.png'
+      end
+
+      def output_file
+        @output_file ||= if @options.output_path
+          File.join(@options.output_path, output_base)
+        else
+          output_base
+        end
+      end
+
+    end
+
+  end
+end
+

data/lib/pipely/actions/graph_live_pipeline.rb

@@ -0,0 +1,34 @@
+module Pipely
+  module Actions
+
+    # Graph a deployed pipeline with live execution statuses.
+    #
+    class GraphLivePipeline
+
+      def initialize(options)
+        @options = options
+      end
+
+      def execute
+        live_pipeline = Pipely::LivePipeline.new(@options.pipeline_id)
+        live_pipeline.print_runs_report
+
+        outfile = if @options.latest_run
+          live_pipeline.render_latest_graph(@options.output_path)
+        else
+          live_pipeline.render_graphs(@options.output_path)
+        end
+
+        if @options.json_output
+          $stdout.puts({ :graph => outfile }.to_json)
+        elsif $stdout.tty?
+          $stdout.puts "Generated #{outfile}"
+        else
+          $stdout.puts outfile
+        end
+      end
+
+    end
+
+  end
+end

data/lib/pipely/actions/list_live_pipelines.rb

@@ -0,0 +1,38 @@
+module Pipely
+  module Actions
+
+    # List currently deployed pipelines
+    #
+    class ListLivePipelines
+
+      def initialize(options)
+        @options = options
+      end
+
+      def execute
+        if @options.json_output
+          $stdout.puts pipeline_ids.to_json
+        else
+          $stdout.puts pipeline_ids.map { |pipeline|
+            [ pipeline['name'], pipeline['id'] ].join("\t")
+          }
+        end
+      end
+
+    private
+
+      def pipeline_ids
+        ids = []
+
+        begin
+          result = Fog::AWS[:data_pipeline].list_pipelines
+          ids += result['pipelineIdList']
+        end while (result['hasMoreResults'] && result['marker'])
+
+        ids
+      end
+
+    end
+
+  end
+end

data/lib/pipely/fog_client.rb

@@ -33,13 +33,9 @@ module Pipely
     end
 
     def task_states_by_scheduled_start
-      c = Fog::AWS[:data_pipeline]
-      instances = c.query_objects(pipeline_id, 'INSTANCE')
-      instance_details = c.describe_objects(pipeline_id, instances['ids'])
-
       task_states_by_scheduled_start = {}
 
-      instance_details['pipelineObjects'].each do |pipeline_object|
+      all_instances.each do |pipeline_object|
         component_id = status = scheduled_start = nil
 
         pipeline_object['fields'].each do |field|
@@ -62,5 +58,32 @@ module Pipely
       task_states_by_scheduled_start
     end
 
+  private
+
+    def all_instances
+      c = Fog::AWS[:data_pipeline]
+
+      result = {}
+      pipeline_objects = []
+
+      begin
+        if result['marker']
+          marker = JSON.parse(result['marker'])['primary']
+        end
+
+        result = c.query_objects(
+          pipeline_id,
+          'INSTANCE',
+          marker: result['marker']
+        )
+
+        instance_details = c.describe_objects(pipeline_id, result['ids'])
+        pipeline_objects += instance_details['pipelineObjects']
+
+      end while (result['hasMoreResults'] && result['marker'])
+
+      pipeline_objects
+    end
+
   end
 end

data/lib/pipely/live_pipeline.rb

@@ -26,22 +26,28 @@ module Pipely
       RunsReport.new(@task_states_by_scheduled_start).print
     end
 
-    def render_graphs
-      @task_states_by_scheduled_start.each do |start, task_states|
-        utc_time = Time.now.to_i
-        formatted_start = start.gsub(/[:-]/, '').sub('T', '-')
+    def render_latest_graph(output_path=nil)
+      latest_start = @task_states_by_scheduled_start.keys.max
+      task_states = @task_states_by_scheduled_start[latest_start]
+      render_graph(latest_start, task_states, output_path)
+    end
 
-        filename = "graphs/#{@pipeline_id}-#{formatted_start}-#{utc_time}.png"
+    def render_graphs(output_path=nil)
+      @task_states_by_scheduled_start.map do |start, task_states|
+        render_graph(start, task_states, output_path)
+      end
+    end
 
-        if $stdout.tty?
-          $stdout.puts "Generating #{filename}"
-        else
-          $stdout.puts filename
-        end
+  private
 
-        Pipely.draw(@definition_json, filename, task_states)
-      end
+    def render_graph(start, task_states, output_path)
+      utc_time = Time.now.to_i
+      formatted_start = start.gsub(/[:-]/, '').sub('T', '-')
+
+      output_base = "#{@pipeline_id}-#{formatted_start}-#{utc_time}.png"
+      filename = File.join((output_path || 'graphs'), output_base)
 
+      Pipely.draw(@definition_json, filename, task_states)
     end
 
   end

data/lib/pipely/options.rb

@@ -0,0 +1,47 @@
+require 'optparse'
+
+module Pipely
+
+  # Options for running the CLI
+  #
+  class Options
+
+    attr_accessor :pipeline_id, :input_path, :output_path,
+      :verbose, :automatic_open, :json_output, :latest_run
+
+    def self.parse
+      options = Pipely::Options.new
+
+      OptionParser.new do |opts|
+        opts.banner = "Usage: pipely [options]"
+
+        opts.on("-p", "--pipeline-id PIPELINE_ID",
+          "ID of a live pipeline to visualize with live statuses") do |id|
+          options.pipeline_id = id
+        end
+
+        opts.on("-l", "--latest", "Graph only the latest run") do |latest|
+          options.latest_run = latest
+        end
+
+        opts.on("-i", "--input PATH",
+          "Path to a JSON pipeline definition file to visualize") do |input|
+          options.input_path = input
+        end
+
+        opts.on("-o", "--output PATH",
+          "Local or S3 path to write Graphviz PNG file(s)") do |output|
+          options.output_path = output
+        end
+
+        opts.on("-j", "--json", "Write STDOUT formatted as JSON") do |json|
+          options.json_output = json
+        end
+      end.parse!
+
+      options
+    end
+
+  end
+
+end

data/lib/pipely/s3_writer.rb

@@ -0,0 +1,33 @@
+require 'fog/storage'
+
+module Pipely
+
+  # Writes content from a String to an S3 path
+  #
+  class S3Writer
+
+    def initialize(s3_path)
+      uri = URI.parse(s3_path)
+      @host, @path = uri.host, uri.path.gsub(/^\//,'')
+    end
+
+    def directory
+      storage = Fog::Storage.new({ provider: 'AWS' })
+      directory = storage.directories.detect{ |d| d.key == @host }
+
+      directory or raise("Couldn't find S3 bucket '#{@host}'")
+    end
+
+    def write(content)
+      remote_file = directory.files.create({
+        :key => @path,
+        :body => content,
+        :public => true,
+      })
+
+      remote_file.public_url
+    end
+
+  end
+
+end

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.3.0" unless defined?(::DataPipelineGraphviz::VERSION)
+  VERSION = "0.4.0" unless defined?(::DataPipelineGraphviz::VERSION)
 end

data/spec/lib/pipely_spec.rb

@@ -2,7 +2,7 @@ require 'pipely'
 
 describe Pipely do
   let(:definition_json) { stub }
-  let(:filename) { stub }
+  let(:filename) { 'path/to/graph.png' }
   let(:definition) { stub }
 
   before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Matt Gillooly
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-12 00:00:00.000000000 Z
+date: 2014-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.21.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.20.0
+        version: 1.21.0
 - !ruby/object:Gem::Dependency
   name: unf
   requirement: !ruby/object:Gem::Requirement
@@ -158,6 +158,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/pipely/actions/graph_file_pipeline.rb
+- lib/pipely/actions/graph_live_pipeline.rb
+- lib/pipely/actions/list_live_pipelines.rb
+- lib/pipely/actions.rb
 - lib/pipely/build/daily_scheduler.rb
 - lib/pipely/build/definition.rb
 - lib/pipely/build/environment_config.rb
@@ -174,8 +178,10 @@ files:
 - lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
 - lib/pipely/live_pipeline.rb
+- lib/pipely/options.rb
 - lib/pipely/reference_list.rb
 - lib/pipely/runs_report.rb
+- lib/pipely/s3_writer.rb
 - lib/pipely/shared_examples.rb
 - lib/pipely/tasks/definition.rb
 - lib/pipely/tasks/deploy.rb