azkaban-rb 0.0.4 → 0.0.5

data/example/.gitignore

@@ -0,0 +1,4 @@
+config.yml
+azkaban-rb-test.zip
+job/
+jobs/

data/example/Rakefile

@@ -0,0 +1,73 @@
+$:.unshift(File.expand_path('../lib/'))
+
+require 'azkaban-rb'
+require 'yaml'
+
+raise 'need config.yml' unless File.exists?('config.yml')
+
+config = YAML.load(File.read('config.yml'))
+
+@@job_namespace = 'azkaban-rb-test'
+@@user_name = config["user_name"]
+@@hdfs_root = "/user/#{@@user_name}"
+@@zip_name = "#{@@job_namespace}.zip"
+@@azkaban_path = @@job_namespace
+
+Azkaban::JobFile.output_dir = "jobs/"
+
+job_dir = "job"
+
+desc "Remove all generated files"
+task :clean_job_conf do
+  `rm -rf #{Azkaban::JobFile.output_dir}` if File.exists? Azkaban::JobFile.output_dir
+end
+
+props :base do
+  set "udf.import.list" => "oink.,com.linkedin.pig.,com.linkedin.pig.date.,org.apache.pig.piggybank.,com.linkedin.pig.characters."  
+  set "hadoop.job.ugi" => "#{@@user_name},hadoop"    
+  set "hdfs.default.classpath.dir" => config["hdfs_classpath"]  
+  set "jvm.args" => config["jvm_args"] if config["jvm_args"]
+  set "classpath" => "pig-0.9.0-core.jar,hadoop-lzo-0.4.9.jar"  
+  set "param.job_root" => @@hdfs_root
+end
+
+namespace @@job_namespace.to_sym do
+  
+  pig_job :test do
+    uses "src/test.pig"
+    reads "#{@@hdfs_root}/input.txt", :as => "input"
+    writes "#{@@hdfs_root}/input_grouped.txt", :as => "output"
+  end
+    
+  pig_job :test2 => :test do
+    uses "src/test2.pig"
+    reads "#{@@hdfs_root}/input_grouped.txt", :as => "input"
+    writes "#{@@hdfs_root}/input_ordered.txt", :as => "output"    
+  end
+  
+  command_job :all => :test2 do
+    uses 'echo "Running the final job"'
+  end
+end
+
+task :clean_job_dir do
+  `rm -rf #{job_dir}` if File.exists? "#{job_dir}"
+end
+
+task :clean => [:clean_job_conf,:clean_job_dir] do
+  `rm -rf #{@@zip_name}`
+end
+
+task :zip => [:clean, :base, "#{@@job_namespace}:all".to_sym] do
+  `mkdir #{job_dir}`
+  `cp #{Azkaban::JobFile.output_dir}/* #{job_dir}`
+  `cp *.jar #{job_dir}`
+  `cp -r src #{job_dir}/src`
+  `cd #{job_dir}; zip -r #{@@zip_name} *; cd ..; mv #{job_dir}/#{@@zip_name} .`
+end
+
+task :deploy => :zip do
+  Azkaban.deploy(config['azkaban_uri'], @@azkaban_path, @@zip_name)
+end
+
+task :default => :zip

data/example/bin/pig

@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 
+# The Pig command script
+#
+# Environment Variables
+#
+#     JAVA_HOME                The java implementation to use.    Overrides JAVA_HOME.
+#
+#     PIG_CLASSPATH Extra Java CLASSPATH entries.
+#
+#     PIG_HEAPSIZE    The maximum amount of heap to use, in MB. 
+#                                        Default is 1000.
+#
+#     PIG_OPTS            Extra Java runtime options.
+#
+#     PIG_CONF_DIR    Alternate conf dir. Default is ${PIG_HOME}/conf.
+#
+#     PIG_ROOT_LOGGER The root appender. Default is INFO,console
+#
+#     PIG_HADOOP_VERSION Version of hadoop to run with.    Default is 20 (0.20).
+#
+#     HBASE_CONF_DIR - Optionally, the HBase configuration to run against
+#                      when using HBaseStorage
+
+cygwin=false
+case "`uname`" in
+CYGWIN*) cygwin=true;;
+esac
+debug=false
+
+# filter command line parameter
+for f in $@; do
+     if [[ $f = "-secretDebugCmd" ]]; then
+        debug=true
+     else
+        remaining="${remaining} $f"
+     fi
+done
+
+# resolve links - $0 may be a softlink
+this="${BASH_SOURCE-$0}"
+while [ -h "$this" ]; do
+    ls=`ls -ld "$this"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '.*/.*' > /dev/null; then
+        this="$link"
+    else
+        this=`dirname "$this"`/"$link"
+    fi
+done
+
+# convert relative path to absolute path
+bin=`dirname "$this"`
+script=`basename "$this"`
+bin=`unset CDPATH; cd "$bin"; pwd`
+this="$bin/$script"
+
+# the root of the Pig installation
+export PIG_HOME=`dirname "$this"`/..
+
+#check to see if the conf dir is given as an optional argument
+if [ $# -gt 1 ]
+then
+    if [ "--config" = "$1" ]
+    then
+        shift
+        confdir=$1
+        shift
+        PIG_CONF_DIR=$confdir
+    fi
+fi
+ 
+# Allow alternate conf dir location.
+PIG_CONF_DIR="${PIG_CONF_DIR:-$PIG_HOME/conf}"
+
+if [ -f "${PIG_CONF_DIR}/pig-env.sh" ]; then
+    . "${PIG_CONF_DIR}/pig-env.sh"
+fi
+
+# some Java parameters
+if [ "$JAVA_HOME" != "" ]; then
+    #echo "run java in $JAVA_HOME"
+    JAVA_HOME=$JAVA_HOME
+fi
+    
+if [ "$JAVA_HOME" = "" ]; then
+    echo "Error: JAVA_HOME is not set."
+    exit 1
+fi
+
+JAVA=$JAVA_HOME/bin/java
+JAVA_HEAP_MAX=-Xmx1000m 
+
+# check envvars which might override default args
+if [ "$PIG_HEAPSIZE" != "" ]; then
+    JAVA_HEAP_MAX="-Xmx""$PIG_HEAPSIZE""m"
+fi
+
+# CLASSPATH initially contains $PIG_CONF_DIR
+CLASSPATH="${PIG_CONF_DIR}"
+CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar
+
+# for developers, add Pig classes to CLASSPATH
+if [ -d "$PIG_HOME/build/classes" ]; then
+    CLASSPATH=${CLASSPATH}:$PIG_HOME/build/classes
+fi
+if [ -d "$PIG_HOME/build/test/classes" ]; then
+    CLASSPATH=${CLASSPATH}:$PIG_HOME/build/test/classes
+fi
+
+# so that filenames w/ spaces are handled correctly in loops below
+IFS=
+
+# for releases, add core pig to CLASSPATH
+for f in $PIG_HOME/pig-*-core.jar; do
+    CLASSPATH=${CLASSPATH}:$f;
+done
+
+# during development pig jar might be in build
+for f in $PIG_HOME/build/pig-*-SNAPSHOT.jar; do
+    CLASSPATH=${CLASSPATH}:$f;
+done
+
+# Set the version for Hadoop, default to 17
+PIG_HADOOP_VERSION="${PIG_HADOOP_VERSION:-20}"
+# add libs to CLASSPATH.    There can be more than one version of the hadoop
+# libraries in the lib dir, so don't blindly add them all.    Only add the one
+# that matche PIG_HADOOP_VERSION.
+for f in $PIG_HOME/lib/*.jar; do
+    filename=`basename $f`
+    IS_HADOOP=`echo $filename | grep hadoop`
+    if [ "${IS_HADOOP}x" == "x" ]; then
+        CLASSPATH=${CLASSPATH}:$f;
+    else 
+        IS_RIGHT_VER=`echo $f | grep hadoop${PIG_HADOOP_VERSION}.jar`
+        if [ "${IS_RIGHT_VER}x" != "x" ]; then
+            CLASSPATH=${CLASSPATH}:$f;
+        fi
+    fi
+done
+
+# if using HBase, likely want to include HBase config
+HBASE_CONF_DIR=${HBASE_CONF_DIR:-/etc/hbase}
+if [ -n "$HBASE_CONF_DIR" ] && [ -d "$HBASE_CONF_DIR" ]; then
+  CLASSPATH=$HBASE_CONF_DIR:$CLASSPATH
+fi
+
+# add user-specified CLASSPATH last
+if [ "$PIG_CLASSPATH" != "" ]; then
+    CLASSPATH=${CLASSPATH}:${PIG_CLASSPATH}
+fi
+
+# default log directory & file
+if [ "$PIG_LOG_DIR" = "" ]; then
+    PIG_LOG_DIR="$PIG_HOME/logs"
+fi
+if [ "$PIG_LOGFILE" = "" ]; then
+    PIG_LOGFILE='pig.log'
+fi
+
+# cygwin path translation
+if $cygwin; then
+    CLASSPATH=`cygpath -p -w "$CLASSPATH"`
+    PIG_HOME=`cygpath -d "$PIG_HOME"`
+    PIG_LOG_DIR=`cygpath -d "$PIG_LOG_DIR"`
+fi
+ 
+# restore ordinary behaviour
+unset IFS
+
+CLASS=org.apache.pig.Main
+
+PIG_OPTS="$PIG_OPTS -Dpig.log.dir=$PIG_LOG_DIR"
+PIG_OPTS="$PIG_OPTS -Dpig.log.file=$PIG_LOGFILE"
+PIG_OPTS="$PIG_OPTS -Dpig.home.dir=$PIG_HOME"
+PIG_OPTS="$PIG_OPTS -Dpig.root.logger=${PIG_ROOT_LOGGER:-INFO,console,DRFA}"
+
+# run it
+if [ "$debug" == "true" ]; then
+    echo "dry run:"
+    echo "$JAVA" $JAVA_HEAP_MAX $PIG_OPTS -classpath "$CLASSPATH" $CLASS ${remaining}
+    echo
+else
+    exec "$JAVA" $JAVA_HEAP_MAX $PIG_OPTS -classpath "$CLASSPATH" $CLASS ${remaining}
+fi

data/example/data/input.txt

@@ -0,0 +1,100 @@
+A	15
+C	18
+C	84
+B	45
+C	16
+E	46
+E	32
+D	63
+C	71
+B	87
+A	76
+D	61
+E	18
+A	69
+A	53
+D	2
+E	60
+A	69
+A	2
+E	23
+B	63
+C	47
+D	89
+B	74
+B	31
+A	81
+A	26
+B	99
+E	72
+B	38
+D	12
+E	26
+D	1
+B	64
+E	21
+C	24
+D	68
+B	82
+D	80
+E	94
+A	98
+D	85
+A	90
+C	13
+B	95
+D	96
+B	74
+A	96
+E	1
+A	11
+E	28
+C	21
+A	15
+C	64
+A	53
+C	57
+A	58
+E	89
+D	42
+E	43
+C	74
+C	93
+C	29
+A	94
+D	79
+D	33
+E	82
+C	13
+C	91
+D	82
+D	31
+D	19
+A	11
+B	90
+C	52
+C	46
+D	43
+D	78
+C	8
+D	94
+B	58
+D	22
+D	40
+A	11
+A	77
+B	84
+C	8
+A	44
+E	94
+C	42
+E	1
+B	78
+A	55
+D	10
+C	88
+E	83
+E	10
+B	95
+E	43
+B	20

data/example/example_config.yml

@@ -0,0 +1,4 @@
+user_name: TODO
+jvm_args: TODO
+hdfs_classpath: TODO
+azkaban_uri: TODO

data/example/src/test.pig

@@ -0,0 +1,7 @@
+rmf $output
+inp = load '$input' as (type:chararray,count:int);
+
+inp_grouped = GROUP inp BY type;
+inp_grouped = FOREACH inp_grouped GENERATE group as type, SUM(inp.count) as count;
+
+store inp_grouped into '$output';

data/example/src/test2.pig

@@ -0,0 +1,6 @@
+rmf $output
+inp = load '$input' as (type:chararray,count:int);
+
+inp_ordered = ORDER inp BY count DESC;
+
+STORE inp_ordered INTO '$output';

data/lib/azkaban-rb/tasks.rb

@@ -1,5 +1,11 @@
 require 'httpclient'
 
+module Rake
+  class Task
+    attr_accessor :job
+  end
+end
+
 module Azkaban
     
   def self.deploy(uri, path, zip_file)
@@ -57,13 +63,17 @@ module Azkaban
   HTTP::Message.mime_type_handler = Proc.new { |path| Azkaban::mime_type_handler(path) }
 
   class JobFile
+    attr_reader :read_locks, :write_locks, :task, :uses
 
     @output_dir = "conf/"
 
     def initialize(task, ext)
+      task.job = self
       @task = task
       @ext = ext
       @args = {}
+      @read_locks = []
+      @write_locks = []
     end
 
     class << self
@@ -76,6 +86,16 @@ module Azkaban
       end
     end
 
+    def reads(name, *options)    
+      @read_locks << name
+      handle_read_write_options(options, name)
+    end
+    
+    def writes(name, *options)
+      @write_locks << name
+      handle_read_write_options(options, name)
+    end
+
     def write
       if @args.size > 0
         file_name = @task.name.gsub(":", "-") + @ext
@@ -94,18 +114,74 @@ module Azkaban
 
     private 
 
+    def handle_read_write_options(options, name)
+      options = options[0] if options.size > 0
+      if options && options.instance_of?(Hash) && options[:as]
+        set "param.#{options[:as]}" => name
+      end
+    end
+
     def create_properties_file(file_name, props)
       unless File.exists? Azkaban::JobFile.output_dir
         Dir.mkdir Azkaban::JobFile.output_dir
       end
       file = File.new(Azkaban::JobFile.output_dir + file_name, "w+")
+      if @read_locks && @read_locks.size > 0
+        file.write("read.lock=#{@read_locks.join(",")}\n")
+      end
+      if @write_locks && @write_locks.size > 0
+        file.write("write.lock=#{@write_locks.join(",")}\n")
+      end
       props.each do |k,v|
         file.write("#{k}=#{v}\n")
       end
       file.close
     end
   end
+  
+  class PigJob < JobFile
+    def initialize(task, ext)
+      super(task,ext)
+      set "type"=>"pig"
+    end
+    
+    def uses(name)
+      set "pig.script"=>name
+    end
+  end
+  
+  class JavaJob < JobFile
+    def initialize(task, ext)
+      super(task,ext)
+      set "type"=>"java"
+    end
+    
+    def uses(name)
+      set "job.class"=>name
+    end
+  end
+
+  class JavaProcessJob < JobFile    
+    def initialize(task, ext)
+      super(task,ext)
+      set "type"=>"java"
+    end
+    
+    def uses(name)
+      set "java.class"=>name
+    end
+  end
 
+  class CommandJob < JobFile    
+    def initialize(task, ext)
+      super(task,ext)
+      set "type"=>"command"
+    end
+    
+    def uses(text)
+      set "command"=>text
+    end
+  end
 end
 
 def props(*args, &b)
@@ -118,12 +194,29 @@ def props(*args, &b)
   end
 end
 
-def job(*args,&b)
-  task(*args) do |t|
-    unless b.nil?
-      job = Azkaban::JobFile.new(t, ".job")
-      job.instance_eval(&b)
-      job.write
-    end
+def job(*args,&b)  
+  make_job(Azkaban::JobFile, args, b)
+end
+
+def pig_job(*args,&b) 
+  make_job(Azkaban::PigJob, args, b)
+end
+
+def java_job(*args,&b) 
+  make_job(Azkaban::JavaJob, args, b)
+end
+
+def java_process_job(*args,&b) 
+  make_job(Azkaban::JavaProcessJob, args, b)
+end
+
+def command_job(*args,&b) 
+  make_job(Azkaban::CommandJob, args, b)
+end
+
+def make_job(job_class,args,b)
+  job = job_class.new(task(*args) { job.write }, ".job")
+  unless b.nil?    
+    job.instance_eval(&b)
   end
 end

data/lib/azkaban-rb/version.rb

@@ -1,5 +1,5 @@
 module Azkaban
   module Rb
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

metadata

@@ -1,8 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: azkaban-rb
 version: !ruby/object:Gem::Version 
+  hash: 21
   prerelease: 
-  version: 0.0.4
+  segments: 
+  - 0
+  - 0
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Matt Hayes
@@ -10,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-23 00:00:00 -07:00
-default_executable: 
+date: 2011-08-20 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: httpclient
@@ -21,6 +25,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 1
+        - 6
         version: 2.1.6
   type: :runtime
   version_requirements: *id001
@@ -38,10 +47,18 @@ files:
 - Gemfile
 - Rakefile
 - azkaban-rb.gemspec
+- example/.gitignore
+- example/Rakefile
+- example/bin/pig
+- example/data/input.txt
+- example/example_config.yml
+- example/hadoop-lzo-0.4.9.jar
+- example/pig-0.9.0-core.jar
+- example/src/test.pig
+- example/src/test2.pig
 - lib/azkaban-rb.rb
 - lib/azkaban-rb/tasks.rb
 - lib/azkaban-rb/version.rb
-has_rdoc: true
 homepage: https://github.com/matthayes/azkaban-rb
 licenses: []
 
@@ -55,17 +72,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: azkaban-rb
-rubygems_version: 1.6.2
+rubygems_version: 1.8.7
 signing_key: 
 specification_version: 3
 summary: Azkaban job generation using Ruby