azkaban-rb 0.0.5 → 0.0.6

data/example/Rakefile

@@ -38,7 +38,7 @@ namespace @@job_namespace.to_sym do
     reads "#{@@hdfs_root}/input.txt", :as => "input"
     writes "#{@@hdfs_root}/input_grouped.txt", :as => "output"
   end
-    
+      
   pig_job :test2 => :test do
     uses "src/test2.pig"
     reads "#{@@hdfs_root}/input_grouped.txt", :as => "input"
@@ -70,4 +70,16 @@ task :deploy => :zip do
   Azkaban.deploy(config['azkaban_uri'], @@azkaban_path, @@zip_name)
 end
 
-task :default => :zip
+task :default => :zip
+
+# Create a run task for each pig job so we can run using Rake.  Parameter substituion is done automatically.
+Rake.application.tasks.find_all do |task|
+  if task.job && task.job.instance_of?(Azkaban::PigJob)
+    desc "Run pig job #{task.name}"
+    task("run:#{task.name}".to_sym) do
+      script = task.job["pig.script"]
+      parameters = task.job.parameters.map { |k,v| "-param #{k}=#{v}"}.join(" ")
+      `bin/pig #{parameters} #{script}`
+    end
+  end
+end

data/lib/azkaban-rb/tasks.rb

@@ -79,6 +79,14 @@ module Azkaban
     class << self
       attr_accessor :output_dir
     end
+    
+    def [](k)
+      @args[k]
+    end
+    
+    def []=(k,v)
+      @args[k] = v
+    end
 
     def set(params)
       params.each do |k,v|
@@ -115,10 +123,7 @@ module Azkaban
     private 
 
     def handle_read_write_options(options, name)
-      options = options[0] if options.size > 0
-      if options && options.instance_of?(Hash) && options[:as]
-        set "param.#{options[:as]}" => name
-      end
+      # nothing to do
     end
 
     def create_properties_file(file_name, props)
@@ -140,14 +145,26 @@ module Azkaban
   end
   
   class PigJob < JobFile
+    attr_reader :parameters
+    
     def initialize(task, ext)
       super(task,ext)
       set "type"=>"pig"
+      @parameters = {}
     end
     
     def uses(name)
       set "pig.script"=>name
     end
+    
+    def handle_read_write_options(options, name)
+      options = options[0] if options.size > 0
+      if options && options.instance_of?(Hash) && options[:as]
+        # set the pig parameter
+        set "param.#{options[:as]}" => name
+        @parameters[options[:as]] = name
+      end
+    end
   end
   
   class JavaJob < JobFile

data/lib/azkaban-rb/version.rb

@@ -1,5 +1,5 @@
 module Azkaban
   module Rb
-    VERSION = "0.0.5"
+    VERSION = "0.0.6"
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: azkaban-rb
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Matt Hayes
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-20 00:00:00 Z
+date: 2011-08-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: httpclient