mandy 0.3.7 → 0.3.9

data/bin/mandy-hadoop

@@ -3,7 +3,7 @@ require "rubygems"
 require "mandy"
 require 'optparse'
 require 'ostruct'
-require 'uri'
+require 'cgi'
 
 options = OpenStruct.new
 
@@ -23,7 +23,7 @@ OptionParser.new do |opts|
   end
   
   opts.on("-j", '--json {"key":"1 value"}', "Pass JSON encoded parameters to jobs") do |config|
-    options.cmdenv = "json=#{URI.encode(config)}"
+    options.cmdenv = "json=#{CGI.escape(config)}"
   end
   
   opts.on("-g", '--gemfile filepath', "Path to your jobs Gemfile (defaults to ./Gemfile)") do |config|

data/lib/dsl.rb

@@ -1,10 +1,7 @@
 module Mandy
   module DSL
     def job(name, &blk)
-      job = Mandy::Job.new(name)
-      job.instance_eval(&blk) unless blk.nil?
-      Mandy::Job.jobs << job
-      job
+      raise "Mandy::DSL has been deprecated please use Mandy.job instead"
     end
   end
 end

data/lib/mandy.rb

@@ -1,6 +1,7 @@
 require "rubygems"
 require "json"
 require "uri"
+require "cgi"
 
 %w(
   support/formatting
@@ -32,4 +33,13 @@ module Mandy
       @stores||={}
     end
   end
+  
+  
+  def job(name, &blk)
+    job = Mandy::Job.new(name)
+    job.instance_eval(&blk) unless blk.nil?
+    Mandy::Job.jobs << job
+    job
+  end
+  module_function :job
 end

data/lib/task.rb

@@ -49,7 +49,7 @@ module Mandy
     end
     
     def find_json_param(name)
-      @json_args ||= JSON.parse(URI.decode(ENV[JSON_PAYLOAD_KEY]))
+      @json_args ||= JSON.parse(CGI.unescape(ENV[JSON_PAYLOAD_KEY]))
       @json_args[name.to_s]
     end
     

data/lib/wrappers/mandy_local_wrapper.rb

@@ -0,0 +1,57 @@
+module Mandy
+  module Local
+    module Wrapper
+      SESSION_ID = Process.pid
+  
+      def set_mandy_config(file_path)
+        @@config_path = file_path
+      end
+  
+      def run_mandy(script, input_files, options = {})
+        begin
+          input_file = concat_input_files(input_files)
+          output_file_path = run_mandy_local(script, input_file, options)
+          return output_file_path unless block_given?
+          #if a block is given then yield the output file path and then delete this file before returning
+          yield output_file_path
+        ensure 
+          File.delete(input_file) if File.exists?(input_file)
+          File.delete(output_file_path) if output_file_path && File.exists?(output_file_path) if block_given?
+        end
+      end
+  
+      private
+        def concat_input_files(inputs)
+          inputs = [inputs] unless inputs.is_a?(Array)
+          base_dir = File.dirname(inputs.first.path)
+          input_file = "#{base_dir}/#{SESSION_ID}.csv"
+          `cat #{inputs.collect{|f| f.path}.join(' ')} > #{input_file}`
+          input_file
+        end
+    
+        def run_mandy_local(script, input, options)
+          mandy_job_params = options.include?(:parameters) ? options[:parameters] : {}
+          param_args = "export json='#{mandy_job_params.to_json}' &&"
+          
+          if options.include?(:lib)
+            FileUtils.cp(script, options[:lib])
+            script = File.join(options[:lib], File.basename(script))
+          end
+
+          output_file = `#{param_args} mandy-local #{script} #{input} #{generate_output_path}`
+          output_file = output_file.split("\n").last
+          output_file
+        ensure
+          File.delete(script) if options.include?(:lib)
+        end
+    
+        def generate_output_path
+          output_dir = "/tmp/mandy_test_output"
+          FileUtils.mkdir_p(output_dir)
+          "#{output_dir}/#{SESSION_ID}"
+        end
+    end
+  end
+end
+
+Object.send(:include, Mandy::Local::Wrapper)

data/lib/wrappers/mandy_wrapper.rb

@@ -0,0 +1,73 @@
+module Mandy
+  module Wrapper
+    SESSION_ID = Process.pid
+  
+    def set_mandy_config(file_path)
+      @@config_path = file_path
+    end
+  
+    def run_mandy(script, input_files, options = {})
+      begin
+        hdfs_path = "#{self.class.to_s.split('::').join('-').downcase}/#{SESSION_ID}"
+        put_files_on_hdfs(hdfs_path, input_files)
+        run_mandy_hadoop(hdfs_path, script, options)
+        output_file_path = get_file_from_hdfs(hdfs_path, options)
+        return output_file_path unless block_given?
+        #if a block is given then yield the output file path and then delete this file before returning
+        yield output_file_path
+      ensure 
+        File.delete(output_file_path) if output_file_path && File.exists?(output_file_path) if block_given?
+      end
+    end
+  
+    private
+      def put_files_on_hdfs(hdfs_path, input_files)
+        input_files = [input_files] unless input_files.is_a?(Array)
+        input_files.each do |input_file|
+          input_file_path = File.expand_path(input_file.path)
+          base_filename = input_file_path.split("/").last
+          dest_file = ["input/#{hdfs_path}", base_filename].join("/")
+          run_command "mandy-put #{input_file_path} #{dest_file}"
+        end
+      end
+    
+      def run_mandy_hadoop(hdfs_path, script, options)
+        mandy_job_params = options.include?(:parameters) ? options[:parameters] : {}
+        param_args = "-j '#{mandy_job_params.to_json}'"
+        param_args += " -p '#{options[:lib]}'" if options.include?(:lib)
+
+        hdfs_output_path = "output/#{hdfs_path}"
+        run_command "mandy-rm output/#{hdfs_path}"
+        run_command "mandy-hadoop #{script} input/#{hdfs_path} output/#{hdfs_path} #{param_args}"
+      end
+    
+      def get_file_from_hdfs(hdfs_path, options)
+        output_file_path = options[:output_file] || generate_output_path
+        hdfs_output_path = "output/#{hdfs_path}"
+        run_command "mandy-get #{get_hdfs_output(hdfs_output_path)} #{output_file_path}"
+        run_command "mandy-rm input/#{hdfs_path}"
+        run_command "mandy-rm output/#{hdfs_path}"
+        output_file_path
+      end
+  
+      def run_command(command)
+        command = "#{command} -c #{@@config_path}"
+        respond_to?(:logger) ? logger.info(command) : p(command)
+        @output = `#{command}`
+      end
+    
+      def generate_output_path
+        output_dir = "/tmp/mandy_output"
+        FileUtils.mkdir_p(output_dir)
+        "#{output_dir}/#{SESSION_ID}"
+      end
+      
+      def get_hdfs_output(hdfs_output_path)
+        @output.each_line do |line|
+          return line.chomp.strip if line.include?(hdfs_output_path)
+        end
+      end
+  end
+end
+
+Object.send(:include, Mandy::Wrapper)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mandy
 version: !ruby/object:Gem::Version 
-  version: 0.3.7
+  version: 0.3.9
 platform: ruby
 authors: 
 - Andy Kent
@@ -76,6 +76,8 @@ files:
 - lib/ruby-hbase/version.rb
 - lib/ruby-hbase/xml_decoder.rb
 - lib/test_runner.rb
+- lib/wrappers/mandy_wrapper.rb
+- lib/wrappers/mandy_local_wrapper.rb
 has_rdoc: true
 homepage: http://github.com/trafficbroker/mandy
 licenses: []