pipely 0.0.1

data/README.md

@@ -0,0 +1,4 @@
+pipely
+======
+
+Visualize pipeline definitions for AWS Data Pipeline

data/Rakefile

@@ -0,0 +1,21 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+require 'cane/rake_task'
+
+RSpec::Core::RakeTask.new do |t|
+    t.pattern = 'spec/**/*_spec.rb'
+end
+
+Cane::RakeTask.new(:quality) do |cane|
+    cane.canefile = '.cane'
+end
+
+task :default => [:spec, :quality]

data/bin/pipely

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'pipely'
+require 'optparse'
+
+output_path = nil
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: pipely [options] definition.json"
+
+  opts.on("-o", "--output PATH",
+          "Path to write PNG file(s) of graph visualization(s)") do |output|
+    output_path = output
+  end
+end.parse!
+
+definition_file = ARGV.last
+definition_json = File.open(definition_file).read
+
+output_base = File.basename(definition_file,".*") + '.png'
+output_file = output_path ? File.join(output_path, output_base) : output_base
+
+puts "Generating #{output_file}"
+Pipely.draw(definition_json, output_file)

data/lib/pipely.rb

@@ -0,0 +1,20 @@
+require 'pipely/definition'
+require 'pipely/graph_builder'
+
+# The top-level module for this gem. It provides the recommended public
+# interface for using Pipely to visualize and manipulate your Data Pipeline
+# definitions.
+#
+module Pipely
+
+  def self.draw(definition_json, filename, node_attributes=nil)
+    definition = Definition.parse(definition_json)
+    definition.apply_node_attributes(node_attributes) if node_attributes
+
+    graph_builder = GraphBuilder.new
+
+    graph = graph_builder.build(definition.components_for_graph)
+    graph.output( :png => filename )
+  end
+
+end

data/lib/pipely/component.rb

@@ -0,0 +1,111 @@
+require 'virtus'
+require 'pipely/reference_list'
+
+module Pipely
+
+  # Represents a Component within a Data Pipeline Definition
+  # http://amzn.to/16lbBKx
+  #
+  class Component
+
+    REFERENCE_KEYS = [
+      'dependsOn',
+      'input',
+      'output',
+      'runsOn',
+      'schedule',
+      'onFail',
+      'onSuccess',
+      'dataFormat',
+    ]
+
+    STATE_COLORS = {
+      'FINISHED' => 'deepskyblue1',
+      'RUNNING' => 'chartreuse',
+      'WAITING_ON_DEPENDENCIES' => 'gray',
+      'WAITING_FOR_RUNNER' => 'bisque4',
+      'FAILED' => 'orangered',
+    }
+
+    include Virtus
+
+    attribute :id, String
+    attribute :type, String
+    attribute :color, String
+    attribute :execution_state, String
+
+    attribute :dependsOn, ReferenceList
+    attribute :input, ReferenceList
+    attribute :output, ReferenceList
+    attribute :runsOn, ReferenceList
+    attribute :schedule, ReferenceList
+    attribute :onFail, ReferenceList
+    attribute :onSuccess, ReferenceList
+    attribute :dataFormat, ReferenceList
+
+    def initialize(args)
+      @original_args = args.clone
+      super
+      coerce_references
+    end
+
+    def coerce_references
+      REFERENCE_KEYS.each do |key|
+        value = send(key)
+        unless value.is_a?(ReferenceList)
+          send("#{key}=", ReferenceList.new(value))
+        end
+      end
+    end
+
+    def graphviz_options
+      {
+        :shape => 'record',
+        :label => "{#{label}}",
+        :color => color || 'black',
+        :fillcolor => STATE_COLORS[execution_state] || 'white',
+        :style => 'filled',
+      }
+    end
+
+    def dependencies(scope=nil)
+      deps = dependsOn.build_dependencies('dependsOn') +
+        input.build_dependencies('input') +
+        output.build_dependencies('output')
+
+      if :all == scope
+        deps += runsOn.build_dependencies(:runsOn)
+        deps += schedule.build_dependencies(:schedule)
+        deps += onFail.build_dependencies(:onFail)
+        deps += onSuccess.build_dependencies(:onSuccess)
+        deps += dataFormat.build_dependencies(:dataFormat)
+      end
+
+      deps
+    end
+
+    def to_json(options={})
+      h = @original_args
+
+      REFERENCE_KEYS.each do |key|
+        value = send(key)
+
+        if value.present?
+          h[key] = value
+        else
+          h.delete(key)
+        end
+      end
+
+      h.to_json(options)
+    end
+
+  private
+
+    def label
+      [id, type, execution_state].compact.join('|')
+    end
+
+  end
+
+end

data/lib/pipely/definition.rb

@@ -0,0 +1,72 @@
+require 'json'
+require 'pipely/component'
+
+module Pipely
+
+  # Pipely's representation of a Pipeline Definition for AWS Data Pipeline
+  # http://amzn.to/1bpW8Ru
+  #
+  class Definition
+
+    # Showing all component types leads to an unwieldy graph.
+    # TODO: make this list configurable.
+    NON_GRAPH_COMPONENT_TYPES = [
+      'Schedule',
+      'SnsAlarm',
+      'Ec2Resource',
+      'EmrCluster',
+      'CSV',
+      nil,
+    ]
+
+    def self.parse(content)
+      objects = JSON.parse(content)['objects']
+      components = objects.map{|obj| Component.new(obj)}
+
+      new(components)
+    end
+
+    def initialize(components)
+      @components = components
+    end
+
+    attr_reader :components
+
+    def components_for_graph
+      components.reject { |component|
+        NON_GRAPH_COMPONENT_TYPES.include?(component['type'])
+      }
+    end
+
+    def to_json
+      { :objects => components }.to_json
+    end
+
+    def apply_component_attributes(component_attributes)
+      self.components.each do |component|
+        if attributes = component_attributes[component.id]
+          component.attributes = attributes
+        end
+      end
+    end
+
+  private
+
+    def get_components(target_component_ids)
+      components.select { |component|
+        target_component_ids.include?(component.id)
+      }
+    end
+
+    def dependencies_of(selected_components)
+      all_dependencies = selected_components.map { |component|
+        component.dependencies(:all)
+      }.flatten.uniq
+
+      Set.new(get_components(all_dependencies.map(&:target_id)))
+    end
+
+  end
+
+end
+

data/lib/pipely/dependency.rb

@@ -0,0 +1,14 @@
+module Pipely
+
+  # Represents a dependency from one Component on another
+  # http://amzn.to/16lbBKx
+  #
+  class Dependency < Struct.new(:label, :target_id, :color)
+
+    def color
+      super || 'black'
+    end
+
+  end
+
+end

data/lib/pipely/graph_builder.rb

@@ -0,0 +1,51 @@
+require 'json'
+require 'graphviz'
+
+module Pipely
+
+  # Builds a GraphViz graph from a set of Components and their Dependencies
+  class GraphBuilder
+
+    def initialize(graph=nil)
+      @graph = graph || GraphViz.new(:G, :type => :digraph)
+    end
+
+    def build(components)
+      add_nodes(components)
+      add_edges(components)
+      @graph
+    end
+
+  private
+
+    # Represent Components as nodes on the graph
+    def add_nodes(components)
+      components.each do |component|
+        @graph.add_nodes(component.id, component.graphviz_options)
+      end
+    end
+
+    # Represent Dependencies as edges on the graph
+    def add_edges(components)
+      components.each do |component|
+        component.dependencies.each do |dependency|
+          options = {
+            :label => dependency.label,
+            :color => dependency.color,
+          }
+
+          options[:dir] = 'back' if ('input' == dependency.label)
+
+          @graph.add_edges(
+            component.id,
+            dependency.target_id,
+            options
+          )
+        end
+      end
+    end
+
+  end
+
+end
+

data/lib/pipely/reference_list.rb

@@ -0,0 +1,32 @@
+require 'pipely/dependency'
+
+module Pipely
+
+  # A list of references to Components for managing dependencies
+  #
+  class ReferenceList
+
+    def initialize(input)
+      @raw_references = [input].flatten.compact
+    end
+
+    def build_dependencies(label)
+      @raw_references.map{|h| Dependency.new(label, h['ref'])}
+    end
+
+    def to_json(options={})
+      if 1 == @raw_references.count
+        @raw_references.first.to_json(options)
+      else
+        @raw_references.to_json(options)
+      end
+    end
+
+    def present?
+      !@raw_references.empty?
+    end
+
+  end
+
+end
+

data/lib/pipely/version.rb

@@ -0,0 +1,3 @@
+module Pipely
+  VERSION = "0.0.1" unless defined?(::DataPipelineGraphviz::VERSION)
+end

data/spec/lib/pipely/component_spec.rb

@@ -0,0 +1,44 @@
+require 'pipely/component'
+require 'pipely/dependency'
+
+describe Pipely::Component do
+
+  subject {
+    described_class.new(
+      id: 'my-component',
+      type: 'OreoSalad',
+      dependsOn: {'ref' => 'asdf'},
+      input: {'ref' => 'infile'},
+      output: {'ref' => 'outfile'},
+      color: 'yellow',
+      execution_state: 'WAITING_FOR_RUNNER',
+    )
+  }
+
+  it 'coerces dependsOn into a ReferenceList' do
+    expect(subject.dependsOn).to be_a(Pipely::ReferenceList)
+  end
+
+  describe '#graphviz_options' do
+    it 'builds properties for graphviz node representing this component' do
+      expect(subject.graphviz_options).to eq({
+        :shape => 'record',
+        :label => '{my-component|OreoSalad|WAITING_FOR_RUNNER}',
+        :color => 'yellow',
+        :fillcolor => 'bisque4',
+        :style => 'filled',
+      })
+    end
+  end
+
+  describe '#dependencies' do
+    it 'includes dependsOn edges' do
+      expect(subject.dependencies).to eq([
+        Pipely::Dependency.new('dependsOn', 'asdf'),
+        Pipely::Dependency.new('input', 'infile'),
+        Pipely::Dependency.new('output', 'outfile'),
+      ])
+    end
+  end
+
+end

data/spec/lib/pipely/definition_spec.rb

@@ -0,0 +1,56 @@
+require 'pipely/definition'
+
+describe Pipely::Definition do
+
+  subject { described_class.parse(definition_json) }
+
+  let(:definition_json) {
+   <<EOF
+      {
+        "objects": [
+          {
+            "id": "DoStuff",
+            "type": "ShellCommandActivity",
+            "onFail": { "ref": "FailureNotify" }
+          },
+          {
+            "id": "FailureNotify",
+            "type": "SnsAlarm"
+          }
+        ]
+      }
+EOF
+  }
+
+  describe '#components' do
+    it 'builds a Component for each object in the definition JSON' do
+      expect(subject.components.count).to eq(2)
+    end
+  end
+
+  describe '#components_for_graph' do
+    it 'filters out node types we do not want on the graph' do
+      expect(subject.components_for_graph.count).to eq(1)
+    end
+  end
+
+  describe '#to_json' do
+    it 'renders the components as JSON' do
+      original = JSON.parse(definition_json)
+      expect(JSON.parse(subject.to_json)).to eq(original)
+    end
+  end
+
+  describe '#apply_component_attributes' do
+    it 'applies attributes to nodes with matching ids' do
+      subject.apply_component_attributes({
+        'DoStuff' => { color: 'pink' },
+      })
+
+      pink_node = subject.components.detect{|n| n.id == 'DoStuff'}
+
+      expect(pink_node.color).to eq('pink')
+    end
+  end
+
+end

data/spec/lib/pipely/dependency_spec.rb

@@ -0,0 +1,11 @@
+require 'pipely/dependency'
+
+describe Pipely::Dependency do
+
+  describe '#color' do
+    it 'defaults to "black"' do
+      expect(subject.color).to eq('black')
+    end
+  end
+
+end

data/spec/lib/pipely/graph_builder_spec.rb

@@ -0,0 +1,42 @@
+require 'pipely/graph_builder'
+
+describe Pipely::GraphBuilder do
+
+  let(:graph) { stub(:graph) }
+
+  let(:node1) {
+    Pipely::Component.new(
+      :id => '1',
+      :dependsOn => { 'ref' => '2' },
+    )
+  }
+
+  let(:node2) {
+    Pipely::Component.new(
+      :id => '2',
+    )
+  }
+
+  subject { described_class.new(graph) }
+
+  describe '#build' do
+    it 'builds a graph from a list of Components' do
+      graph.should_receive(:add_nodes).
+        with(node1.id, node1.graphviz_options).ordered
+
+      graph.should_receive(:add_nodes).
+        with(node2.id, node2.graphviz_options).ordered
+
+      graph.should_receive(:add_edges).
+        with(
+          node1.id,
+          node2.id,
+          :label => 'dependsOn',
+          :color => 'black',
+        ).ordered
+
+      subject.build([node1, node2])
+    end
+  end
+
+end

data/spec/lib/pipely/reference_list_spec.rb

@@ -0,0 +1,45 @@
+require 'pipely/reference_list'
+
+describe Pipely::ReferenceList do
+
+  context 'given nil input' do
+    subject { described_class.new(nil) }
+
+    describe '#build_dependencies' do
+      it 'returns an empty array' do
+        expect(subject.build_dependencies('dependsOn')).to eq([])
+      end
+    end
+  end
+
+  context 'given a single input' do
+    subject { described_class.new({ 'ref' => 'foo' }) }
+
+    describe '#build_dependencies' do
+      it 'returns an array of the single reference' do
+        expect(subject.build_dependencies('dependsOn')).to eq([
+          Pipely::Dependency.new('dependsOn', 'foo'),
+        ])
+      end
+    end
+  end
+
+  context 'given an array of references as input' do
+    subject {
+      described_class.new([
+        { 'ref' => 'foo' },
+        { 'ref' => 'bar' },
+      ])
+    }
+
+    describe '#build_dependencies' do
+      it 'returns an array of the single reference' do
+        expect(subject.build_dependencies('dependsOn')).to eq([
+          Pipely::Dependency.new('dependsOn', 'foo'),
+          Pipely::Dependency.new('dependsOn', 'bar'),
+        ])
+      end
+    end
+  end
+
+end

data/spec/lib/pipely_spec.rb

@@ -0,0 +1,38 @@
+require 'pipely'
+
+describe Pipely do
+  let(:definition_json) { stub }
+  let(:filename) { stub }
+  let(:definition) { stub }
+
+  before do
+    Pipely::Definition.stub(:parse).with(definition_json) { definition }
+  end
+
+  describe '.draw' do
+    let(:components) { stub }
+    let(:definition) { stub(:definition, :components_for_graph => components) }
+    let(:graph) { stub(:graph, :output => nil) }
+
+    before do
+      Pipely::GraphBuilder.any_instance.stub(:build).with(components) { graph }
+    end
+
+    it 'parses a JSON definition and builds a graph' do
+      graph.should_receive(:output).with(:png => filename)
+
+      described_class.draw(definition_json, filename)
+    end
+
+    context 'with node_attributes' do
+      let(:node_attributes) { stub }
+
+      it 'applies the node_attributes to the definition' do
+        definition.should_receive(:apply_node_attributes).with(node_attributes)
+
+        described_class.draw(definition_json, filename, node_attributes)
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: pipely
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Matt Gillooly
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-graphviz
+  requirement: &70104078435520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70104078435520
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70104078435100 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70104078435100
+- !ruby/object:Gem::Dependency
+  name: virtus
+  requirement: &70104078434680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70104078434680
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70104078434260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70104078434260
+- !ruby/object:Gem::Dependency
+  name: cane
+  requirement: &70104078433840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70104078433840
+description: 
+email:
+- matt@swipely.com
+executables:
+- pipely
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pipely/component.rb
+- lib/pipely/definition.rb
+- lib/pipely/dependency.rb
+- lib/pipely/graph_builder.rb
+- lib/pipely/reference_list.rb
+- lib/pipely/version.rb
+- lib/pipely.rb
+- Rakefile
+- README.md
+- spec/lib/pipely/component_spec.rb
+- spec/lib/pipely/definition_spec.rb
+- spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/graph_builder_spec.rb
+- spec/lib/pipely/reference_list_spec.rb
+- spec/lib/pipely_spec.rb
+- !binary |-
+  YmluL3BpcGVseQ==
+homepage: http://github.com/swipely/pipely
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Generate dependency graphs from pipeline definitions.
+test_files:
+- spec/lib/pipely/component_spec.rb
+- spec/lib/pipely/definition_spec.rb
+- spec/lib/pipely/dependency_spec.rb
+- spec/lib/pipely/graph_builder_spec.rb
+- spec/lib/pipely/reference_list_spec.rb
+- spec/lib/pipely_spec.rb
+has_rdoc: