pipely 0.0.1 → 0.1.0

data/README.md

@@ -1,4 +1,41 @@
 pipely
 ======
+[![Gem Version](https://badge.fury.io/rb/pipely.png)](http://badge.fury.io/rb/pipely) [![Build Status](https://travis-ci.org/swipely/pipely.png?branch=master)](https://travis-ci.org/swipely/pipely) [![Code Climate](https://codeclimate.com/repos/524b941156b1025b6c08a96a/badges/c0ad2bbec610f1d0f0f7/gpa.png)](https://codeclimate.com/repos/524b941156b1025b6c08a96a/feed)
 
 Visualize pipeline definitions for AWS Data Pipeline
+
+"AWS Data Pipeline is a web service that you can use to automate the movement and transformation of data. With AWS Data Pipeline, you can define data-driven workflows, so that tasks can be dependent on the successful completion of previous tasks."
+
+http://docs.aws.amazon.com/datapipeline/latest/DeveloperGuide/what-is-datapipeline.html
+
+
+## Install
+
+(First install [GraphViz](http://www.graphviz.org) if it is not already installed.)
+
+Into Gemfile from rubygems.org:
+
+    gem 'pipely'
+
+Into environment gems from rubygems.org:
+
+    gem install pipely
+
+
+## Usage
+
+(If you used the Gemfile install, prefix the below commands with `bundle exec`.)
+
+### Command-line Interface
+
+To render a JSON pipeline definition as a PNG graph visualization:
+
+    pipely definition.json
+
+To specify the output path for PNG files:
+
+    pipely -o path/to/graph/pngs definition.json
+
+### Rake Tasks
+
+Coming soon.

data/bin/pipely

@@ -3,10 +3,21 @@
 require 'pipely'
 require 'optparse'
 
-output_path = nil
+pipeline_id = input_path = output_path = nil
+verbose = automatic_open = false
 
 OptionParser.new do |opts|
-  opts.banner = "Usage: pipely [options] definition.json"
+  opts.banner = "Usage: pipely [options]"
+
+  opts.on("-p", "--pipeline-id PIPELINE_ID",
+          "ID of a live pipeline to visualize with live execution statuses") do |id|
+    pipeline_id = id
+  end
+
+  opts.on("-i", "--input PATH",
+          "Path to a JSON pipeline definition file to visualize") do |input|
+    input_path = input
+  end
 
   opts.on("-o", "--output PATH",
           "Path to write PNG file(s) of graph visualization(s)") do |output|
@@ -14,11 +25,20 @@ OptionParser.new do |opts|
   end
 end.parse!
 
-definition_file = ARGV.last
-definition_json = File.open(definition_file).read
+if pipeline_id
+  live_pipeline = Pipely::LivePipeline.new(pipeline_id)
+  live_pipeline.print_runs_report
+  live_pipeline.render_graphs
+
+  puts "open: #{automatic_open}"
+  puts "verbose: #{verbose}"
 
-output_base = File.basename(definition_file,".*") + '.png'
-output_file = output_path ? File.join(output_path, output_base) : output_base
+else
+  input_path ||= ARGV.last
+  definition_json = File.open(input_path).read
+  output_base = File.basename(definition_file,".*") + '.png'
+  output_file = output_path ? File.join(output_path, output_base) : output_base
 
-puts "Generating #{output_file}"
-Pipely.draw(definition_json, output_file)
+  puts "Generating #{output_file}"
+  Pipely.draw(definition_json, output_file)
+end

data/lib/pipely.rb

@@ -1,5 +1,6 @@
 require 'pipely/definition'
 require 'pipely/graph_builder'
+require 'pipely/live_pipeline'
 
 # The top-level module for this gem. It provides the recommended public
 # interface for using Pipely to visualize and manipulate your Data Pipeline
@@ -7,9 +8,12 @@ require 'pipely/graph_builder'
 #
 module Pipely
 
-  def self.draw(definition_json, filename, node_attributes=nil)
+  def self.draw(definition_json, filename, component_attributes=nil)
     definition = Definition.parse(definition_json)
-    definition.apply_node_attributes(node_attributes) if node_attributes
+
+    if component_attributes
+      definition.apply_component_attributes(component_attributes)
+    end
 
     graph_builder = GraphBuilder.new
 

data/lib/pipely/component.rb

@@ -27,7 +27,7 @@ module Pipely
       'FAILED' => 'orangered',
     }
 
-    include Virtus
+    include Virtus.model
 
     attribute :id, String
     attribute :type, String
@@ -84,7 +84,7 @@ module Pipely
       deps
     end
 
-    def to_json(options={})
+    def to_json(options={}, depth=0)
       h = @original_args
 
       REFERENCE_KEYS.each do |key|

data/lib/pipely/fog_client.rb

@@ -0,0 +1,66 @@
+require 'fog'
+
+module Pipely
+
+  # Uses Fog to communicate with the AWS Data Pipeline service
+  class FogClient < Struct.new(:pipeline_id)
+
+    def definition
+      objects = Fog::AWS[:data_pipeline].get_pipeline_definition(pipeline_id)
+
+      flattened_objects = []
+
+      objects['pipelineObjects'].each do |object|
+        h = {
+          id: object['id'],
+          name: object['name'],
+        }
+
+        object['fields'].each do |field|
+          k = field['key']
+          if field['refValue']
+            h[k] ||= []
+            h[k] << { ref: field['refValue'] }
+          else
+            h[k] = field['stringValue']
+          end
+        end
+
+        flattened_objects << h
+      end
+
+      { objects: flattened_objects }.to_json
+    end
+
+    def task_states_by_scheduled_start
+      c = Fog::AWS[:data_pipeline]
+      instances = c.query_objects(pipeline_id, 'INSTANCE')
+      instance_details = c.describe_objects(pipeline_id, instances['ids'])
+
+      task_states_by_scheduled_start = {}
+
+      instance_details['pipelineObjects'].each do |pipeline_object|
+        component_id = status = scheduled_start = nil
+
+        pipeline_object['fields'].each do |field|
+          case field['key']
+          when '@componentParent'
+            component_id = field['refValue']
+          when '@status'
+            status = field['stringValue']
+          when '@scheduledStartTime'
+            scheduled_start = field['stringValue']
+          end
+        end
+
+        task_states_by_scheduled_start[scheduled_start] ||= {}
+        task_states_by_scheduled_start[scheduled_start][component_id] = {
+          execution_state: status
+        }
+      end
+
+      task_states_by_scheduled_start
+    end
+
+  end
+end

data/lib/pipely/graph_builder.rb

@@ -29,22 +29,34 @@ module Pipely
     def add_edges(components)
       components.each do |component|
         component.dependencies.each do |dependency|
-          options = {
-            :label => dependency.label,
-            :color => dependency.color,
-          }
-
-          options[:dir] = 'back' if ('input' == dependency.label)
-
-          @graph.add_edges(
-            component.id,
-            dependency.target_id,
-            options
-          )
+          add_edge(component, dependency)
         end
       end
     end
 
+    def add_edge(component, dependency)
+      options = {
+        :label => dependency.label,
+        :color => dependency.color,
+      }
+
+      options[:dir] = 'back' if ('input' == dependency.label)
+
+      if 'output' == dependency.label
+        @graph.add_edges(
+          dependency.target_id,
+          component.id,
+          options
+        )
+      else
+        @graph.add_edges(
+          component.id,
+          dependency.target_id,
+          options
+        )
+      end
+    end
+
   end
 
 end

data/lib/pipely/live_pipeline.rb

@@ -0,0 +1,42 @@
+require 'pipely/fog_client'
+require 'pipely/runs_report'
+
+module Pipely
+
+  # Represent a pipeline that has been deployed to AWS DataPipeline
+  class LivePipeline
+
+    def initialize(pipeline_id)
+      @pipeline_id = pipeline_id
+
+      client = FogClient.new(pipeline_id)
+      @definition_json = client.definition
+      @task_states_by_scheduled_start = client.task_states_by_scheduled_start
+
+      unless @definition_json
+        raise "No definition found for #{client.pipeline_id}"
+      end
+
+      if @task_states_by_scheduled_start.empty?
+        raise "No runs found for #{client.pipeline_id}"
+      end
+    end
+
+    def print_runs_report
+      RunsReport.new(@task_states_by_scheduled_start).print
+    end
+
+    def render_graphs
+      @task_states_by_scheduled_start.each do |start, task_states|
+        utc_time = Time.now.to_i
+        formatted_start = start.gsub(/[:-]/, '').sub('T', '-')
+
+        filename = "graphs/#{@pipeline_id}-#{formatted_start}-#{utc_time}.png"
+        puts "Generating #{filename}"
+        Pipely.draw(@definition_json, filename, task_states)
+      end
+
+    end
+
+  end
+end

data/lib/pipely/reference_list.rb

@@ -14,7 +14,7 @@ module Pipely
       @raw_references.map{|h| Dependency.new(label, h['ref'])}
     end
 
-    def to_json(options={})
+    def to_json(options={}, depth=0)
       if 1 == @raw_references.count
         @raw_references.first.to_json(options)
       else

data/lib/pipely/runs_report.rb

@@ -0,0 +1,20 @@
+module Pipely
+
+  # Prints a CLI report of the execution status of a live pipeline
+  class RunsReport < Struct.new(:task_states_by_scheduled_start)
+
+    def print
+      task_states_by_scheduled_start.each do |scheduled_start, task_states|
+        task_states.to_a.sort_by(&:first).each do |task_name, attributes|
+          current_state = attributes[:execution_state]
+
+          puts task_name.ljust(55) +
+            "scheduled_start: #{scheduled_start}\t\t" +
+            "current_state: #{current_state}"
+        end
+      end
+    end
+
+  end
+end
+

data/lib/pipely/version.rb

@@ -1,3 +1,3 @@
 module Pipely
-  VERSION = "0.0.1" unless defined?(::DataPipelineGraphviz::VERSION)
+  VERSION = "0.1.0" unless defined?(::DataPipelineGraphviz::VERSION)
 end

data/spec/lib/pipely_spec.rb

@@ -24,13 +24,14 @@ describe Pipely do
       described_class.draw(definition_json, filename)
     end
 
-    context 'with node_attributes' do
-      let(:node_attributes) { stub }
+    context 'with component_attributes' do
+      let(:component_attributes) { stub }
 
-      it 'applies the node_attributes to the definition' do
-        definition.should_receive(:apply_node_attributes).with(node_attributes)
+      it 'applies the component_attributes to the definition' do
+        definition.should_receive(:apply_component_attributes).
+          with(component_attributes)
 
-        described_class.draw(definition_json, filename, node_attributes)
+        described_class.draw(definition_json, filename, component_attributes)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pipely
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-02 00:00:00.000000000 Z
+date: 2013-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-graphviz
-  requirement: &70104078435520 !ruby/object:Gem::Requirement
+  requirement: &70261464770320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104078435520
+  version_requirements: *70261464770320
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70104078435100 !ruby/object:Gem::Requirement
+  requirement: &70261464769340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,21 +32,32 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70104078435100
+  version_requirements: *70261464769340
 - !ruby/object:Gem::Dependency
   name: virtus
-  requirement: &70104078434680 !ruby/object:Gem::Requirement
+  requirement: &70261464784920 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70261464784920
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: &70261464783540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.18.0
   type: :runtime
   prerelease: false
-  version_requirements: *70104078434680
+  version_requirements: *70261464783540
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70104078434260 !ruby/object:Gem::Requirement
+  requirement: &70261464782460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70104078434260
+  version_requirements: *70261464782460
 - !ruby/object:Gem::Dependency
   name: cane
-  requirement: &70104078433840 !ruby/object:Gem::Requirement
+  requirement: &70261464781200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +76,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70104078433840
+  version_requirements: *70261464781200
 description: 
 email:
 - matt@swipely.com
@@ -77,8 +88,11 @@ files:
 - lib/pipely/component.rb
 - lib/pipely/definition.rb
 - lib/pipely/dependency.rb
+- lib/pipely/fog_client.rb
 - lib/pipely/graph_builder.rb
+- lib/pipely/live_pipeline.rb
 - lib/pipely/reference_list.rb
+- lib/pipely/runs_report.rb
 - lib/pipely/version.rb
 - lib/pipely.rb
 - Rakefile
@@ -92,7 +106,8 @@ files:
 - !binary |-
   YmluL3BpcGVseQ==
 homepage: http://github.com/swipely/pipely
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -103,12 +118,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -4454073296924687218
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -4454073296924687218
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.11
@@ -122,4 +143,3 @@ test_files:
 - spec/lib/pipely/graph_builder_spec.rb
 - spec/lib/pipely/reference_list_spec.rb
 - spec/lib/pipely_spec.rb
-has_rdoc: