azkaban-rb 0.0.6 → 0.0.7

data/azkaban-rb.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   
   s.add_dependency "httpclient", "~> 2.1.6"
+  s.add_dependency "GraphvizR", "~> 0.5.1"
 end

data/example/Rakefile

@@ -23,7 +23,7 @@ task :clean_job_conf do
 end
 
 props :base do
-  set "udf.import.list" => "oink.,com.linkedin.pig.,com.linkedin.pig.date.,org.apache.pig.piggybank.,com.linkedin.pig.characters."  
+  set "udf.import.list" => "oink.,org.apache.pig.piggybank."  
   set "hadoop.job.ugi" => "#{@@user_name},hadoop"    
   set "hdfs.default.classpath.dir" => config["hdfs_classpath"]  
   set "jvm.args" => config["jvm_args"] if config["jvm_args"]
@@ -72,6 +72,12 @@ end
 
 task :default => :zip
 
+desc "Visualize the data flow (requires GraphViz installed)"
+task :visualize do
+  rakeGraph = RakeGraph.new
+  rakeGraph.visualize("ExampleAzkabanDataflow", "example_azkaban_dataflow.png")
+end
+
 # Create a run task for each pig job so we can run using Rake.  Parameter substituion is done automatically.
 Rake.application.tasks.find_all do |task|
   if task.job && task.job.instance_of?(Azkaban::PigJob)
@@ -82,4 +88,5 @@ Rake.application.tasks.find_all do |task|
       `bin/pig #{parameters} #{script}`
     end
   end
-end
+end
+

data/lib/azkaban-rb.rb

@@ -1,5 +1,6 @@
 require "azkaban-rb/version"
 require "azkaban-rb/tasks"
+require "azkaban-rb/visualization"
 
 module Azkaban
   module Rb

data/lib/azkaban-rb/tasks.rb

@@ -63,7 +63,7 @@ module Azkaban
   HTTP::Message.mime_type_handler = Proc.new { |path| Azkaban::mime_type_handler(path) }
 
   class JobFile
-    attr_reader :read_locks, :write_locks, :task, :uses
+    attr_reader :read_locks, :write_locks, :task, :uses_arg
 
     @output_dir = "conf/"
 
@@ -154,6 +154,7 @@ module Azkaban
     end
     
     def uses(name)
+      @uses_arg = name
       set "pig.script"=>name
     end
     
@@ -174,6 +175,7 @@ module Azkaban
     end
     
     def uses(name)
+      @uses_arg = name
       set "job.class"=>name
     end
   end
@@ -185,6 +187,7 @@ module Azkaban
     end
     
     def uses(name)
+      @uses_arg = name
       set "java.class"=>name
     end
   end
@@ -196,6 +199,7 @@ module Azkaban
     end
     
     def uses(text)
+      @uses_arg = text
       set "command"=>text
     end
   end

data/lib/azkaban-rb/version.rb

@@ -1,5 +1,5 @@
 module Azkaban
   module Rb
-    VERSION = "0.0.6"
+    VERSION = "0.0.7"
   end
 end

data/lib/azkaban-rb/visualization.rb

@@ -0,0 +1,221 @@
+require 'graphviz_r'
+
+class RakeGraph
+  attr_reader :tasks
+  
+  def initialize(namespaces = nil)
+    @namespaces = namespaces
+    @tasks = {}
+    Rake.application.tasks.find_all{ |task| (not task.job.nil?)}.each do |task|
+      tasks[RakeGraph.task_name(task)] = task if (task.job.read_locks.size + task.job.write_locks.size) > 0
+    end
+    @nodes = {}
+    @edges = []
+    construct_graph()
+  end
+  
+  def RakeGraph.task_name(task)
+    task_name = "TASK#{task}"
+    task_name = task_name.gsub(/[^0-9a-z ]/i, '')
+    return task_name
+  end
+  
+  def RakeGraph.data_name(name)
+    name = "DATA"+name.gsub(/[^0-9a-z ]/i, '')
+    return name
+  end
+  
+  def task_in_namespace(task)
+    return true if @namespaces.nil? or @namespaces.size == 0
+    return (task.scope & @namespaces).size > 0
+  end
+  
+  def find_prereq(task, prereq)    
+    scopes = Array.new(task.scope)
+    while prereq.start_with? '^'
+      scopes.pop
+      prereq.slice!(0)
+    end
+    return RakeGraph.task_name(scopes.join('')+prereq)
+  end
+  
+  def construct_graph()
+    # first add all of the task nodes
+    @tasks.each do |task_name, task|
+      next unless task_in_namespace(task)        
+      node = TaskNode.new(task)
+      @nodes[node.name] = node
+    end
+    
+    # now add all of the edges and data nodes    
+    data_nodes = {}
+    @nodes.each do |name, node|
+      task = node.task
+      # find all prereq tasks
+      # task.prerequisites.each do |prereq|
+      #   prereq = find_prereq(task, prereq)
+      #   next unless @nodes.has_key?(prereq)
+      #   @edges << TaskEdge.new(prereq, node.name)
+      # end
+      # find all data reads
+      task.job.read_locks.each do |read_lock|
+        data_name = RakeGraph.data_name(read_lock)
+        data_nodes[data_name] = DataNode.new(read_lock) unless data_nodes.has_key? data_name
+        @edges << DataEdge.new(data_name, node.name)
+      end
+      # find all data writes
+      task.job.write_locks.each do |write_lock|
+        data_name = RakeGraph.data_name(write_lock)
+        data_nodes[data_name] = DataNode.new(write_lock) unless data_nodes.has_key? data_name
+        @edges << DataEdge.new(node.name, data_name)
+      end
+    end    
+    data_nodes.each do |key, value|
+      @nodes[key] = value
+    end
+  end
+  
+  class Node
+    attr_reader :name, :type
+    
+    def initialize(name, type)
+      @name = name
+      @type = type
+    end
+    
+    def fontcolor
+      return '#000000'
+    end
+  
+    def to_s
+      return "#{@type}: #{@name}"
+    end
+  end
+  
+  class TaskNode < Node
+    attr_reader :task
+    
+    def initialize(task)
+      super(RakeGraph.task_name(task), task.job.class.to_s)
+      @task = task
+    end
+    
+    def label
+      label = "<#{@task}<br/>#{@task.job.uses_arg}>"
+      return label.to_sym
+    end
+    
+    def shape
+      return :ellipse
+    end
+    
+    def fillcolor
+      case @type
+        when 'Azkaban::PigJob'
+          return '#e7a5a5'
+        when 'Azkaban::JavaJob'
+          return '#E7C6A5'
+        when 'Azkaban::CommandJob'
+          return '#e7e6a5'
+      end
+      return ""
+    end
+  end
+  
+  class DataNode < Node
+    attr_reader :filename
+    
+    def initialize(filename)
+      super(RakeGraph.data_name(filename), "data")
+      @filename = filename
+    end
+    
+    def label
+      label = @filename
+      return "<#{label}>".to_sym
+    end
+    
+    def shape
+      return :box
+    end
+    
+    def fillcolor
+      return '#d2e3f3'
+    end
+  end
+  
+  class Edge
+    attr_reader :source, :dest, :type
+    
+    def initialize(source, dest)
+      @source = source
+      @dest = dest
+    end
+    
+    def to_s
+      return "#{source} >> #{dest}"
+    end
+  end
+  
+  class TaskEdge < Edge
+    def initialize(source, dest)
+      super(source, dest)
+      @type = "task"
+    end
+    
+    def style
+      return :dotted
+    end
+  end
+  
+  class DataEdge < Edge
+    def initialize(source, dest)
+      super(source, dest)
+      @type = "data"
+    end
+    
+    def style
+      :solid
+    end
+  end
+  
+  def visualize(name, output_file)
+    g = GraphvizR.new name
+    g.graph[:label => name]
+    add_nodes(g)
+    add_edges(g)
+    g.output output_file
+  end
+  
+  def add_nodes(g)
+    @nodes.each do |name, node|
+      g[name] [:label => @label_block.nil? ? node.label : @label_block.call(node), 
+        :shape => @shape_block.nil? ? node.shape : @shape_block.call(node),
+        :fillcolor => @fillcolor_block.nil? ? node.fillcolor : @fillcolor_block.call(node),
+        :style => :filled, 
+        :fontcolor => @fontcolor_block.nil? ? node.fontcolor : @fontcolor_block.call(node)]
+    end
+  end
+  
+  def add_edges(g)
+    @edges.each do |edge|
+      (g[edge.source]>>g[edge.dest])[:style => edge.style]
+    end
+  end
+  
+  def set_label(&block)
+    @label_block = block
+  end
+  
+  def set_fillcolor(&block)
+    @fillcolor_block = block
+  end
+  
+  def set_fontcolor(&block)
+    @fontcolor_block = block
+  end
+  
+  def set_shape(&block)
+    @shape_block = block
+  end
+end

metadata

@@ -1,41 +1,43 @@
 --- !ruby/object:Gem::Specification 
 name: azkaban-rb
 version: !ruby/object:Gem::Version 
-  hash: 19
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 6
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
-- Matt Hayes
+  - Matt Hayes
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-08-22 00:00:00 Z
+date: 2011-08-23 00:00:00 -07:00
+default_executable: 
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: httpclient
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 2
-        - 1
-        - 6
-        version: 2.1.6
-  type: :runtime
-  version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: httpclient
+    prerelease: false
+    requirement: &id001 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 2.1.6
+    type: :runtime
+    version_requirements: *id001
+  - !ruby/object:Gem::Dependency 
+    name: GraphvizR
+    prerelease: false
+    requirement: &id002 !ruby/object:Gem::Requirement 
+      none: false
+      requirements: 
+        - - ~>
+          - !ruby/object:Gem::Version 
+            version: 0.5.1
+    type: :runtime
+    version_requirements: *id002
 description: azkaban-rb allows Azkaban jobs to be modeled as rake tasks
 email: 
-- matthew.terence.hayes@gmail.com
+  - matthew.terence.hayes@gmail.com
 executables: []
 
 extensions: []
@@ -43,22 +45,24 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- .gitignore
-- Gemfile
-- Rakefile
-- azkaban-rb.gemspec
-- example/.gitignore
-- example/Rakefile
-- example/bin/pig
-- example/data/input.txt
-- example/example_config.yml
-- example/hadoop-lzo-0.4.9.jar
-- example/pig-0.9.0-core.jar
-- example/src/test.pig
-- example/src/test2.pig
-- lib/azkaban-rb.rb
-- lib/azkaban-rb/tasks.rb
-- lib/azkaban-rb/version.rb
+  - .gitignore
+  - Gemfile
+  - Rakefile
+  - azkaban-rb.gemspec
+  - example/.gitignore
+  - example/Rakefile
+  - example/bin/pig
+  - example/data/input.txt
+  - example/example_config.yml
+  - example/hadoop-lzo-0.4.9.jar
+  - example/pig-0.9.0-core.jar
+  - example/src/test.pig
+  - example/src/test2.pig
+  - lib/azkaban-rb.rb
+  - lib/azkaban-rb/tasks.rb
+  - lib/azkaban-rb/version.rb
+  - lib/azkaban-rb/visualization.rb
+has_rdoc: true
 homepage: https://github.com/matthayes/azkaban-rb
 licenses: []
 
@@ -66,29 +70,23 @@ post_install_message:
 rdoc_options: []
 
 require_paths: 
-- lib
+  - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
 requirements: []
 
 rubyforge_project: azkaban-rb
-rubygems_version: 1.8.7
+rubygems_version: 1.5.1
 signing_key: 
 specification_version: 3
 summary: Azkaban job generation using Ruby