trafficbroker-mandy 0.1.1 → 0.1.2

data/Rakefile

@@ -8,4 +8,10 @@ task :default => :spec
 Spec::Rake::SpecTask.new(:spec) do |t|
   t.spec_files = FileList['spec/lib/**/*_spec.rb']
   t.spec_opts = %w{-f s -c -L mtime}
+end
+
+task :gem do
+  `sudo gem build mandy.gemspec`
+  `mkdir pkg; mv mandy-*.gem pkg/`
+  `sudo gem install pkg/mandy-*.gem`
 end

data/bin/mandy-hadoop

@@ -7,22 +7,28 @@ end
 file   = ARGV[0]
 filename = File.basename(file)
 input  = ARGV[1]
-output = ARGV[2]
+output_folder = ARGV[2]
 config = ARGV[3]
 
 require absolute_path(file)
 
-jobconf = Mandy::Job.default.settings.map { |key, value| %(-jobconf #{key}="#{value}") }.join(' ')
+Mandy::Job.jobs.each_with_index do |job, i|
+  
+  jobconf = job.settings.map { |key, value| %(-jobconf #{key}="#{value}") }.join(' ')
+  
+  output = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+  
+  command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar \
+                  -additionalconfspec "#{config}" \
+                  -input "#{input}"  \
+                  -mapper "mandy-map #{filename} '#{job.name}'"  \
+                  -reducer "mandy-reduce #{filename} '#{job.name}'"  \
+                  -file "#{file}" \
+                  -output "#{output}" \
+                  #{jobconf})
 
-command = %($HADOOP_HOME/bin/hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar \
-                -additionalconfspec "#{config}" \
-                -input "#{input}"  \
-                -mapper "mandy-map #{filename}"  \
-                -reducer "mandy-reduce #{filename}"  \
-                -file "#{file}" \
-                -output "#{output}" \
-                #{jobconf})
+  `#{command}`
 
-`#{command}`
-
-# puts "#{command}"
+  # puts "#{command}"
+  input = output
+end

data/bin/mandy-local

@@ -1,11 +1,18 @@
 #!/usr/bin/env ruby
 
+require "fileutils"
+
 def absolute_path(path)
   path =~ /^\// ? path : File.join(Dir.pwd, path)
 end
 
 file   = absolute_path(ARGV[0])
 input  = absolute_path(ARGV[1])
-output = absolute_path(ARGV[2])
+output_folder = FileUtils.mkdir_p(absolute_path(ARGV[2]))
+require file
 
-`cat #{input} | mandy-map #{file} | sort | mandy-reduce #{file} > #{output}`
+Mandy::Job.jobs.each_with_index do |job, i|
+  out = File.join(output_folder, "#{i+1}-#{job.name.downcase.gsub(/\W/, '-')}")
+  `cat #{input} | mandy-map #{file} "#{job.name}" | sort | mandy-reduce #{file} "#{job.name}" > #{out}`
+  input = out
+end

data/bin/mandy-map

@@ -5,7 +5,8 @@ def absolute_path(path)
 end
 
 file   = absolute_path(ARGV[0])
+job_name = ARGV[1]
 
 require file
 
-Mandy::Job.default.run_map
+Mandy::Job.find_by_name(job_name).run_map

data/bin/mandy-reduce

@@ -5,7 +5,8 @@ def absolute_path(path)
 end
 
 file   = absolute_path(ARGV[0])
+job_name = ARGV[1]
 
 require file
 
-Mandy::Job.default.run_reduce
+Mandy::Job.find_by_name(job_name).run_reduce

data/lib/dsl.rb

@@ -1,24 +1,10 @@
 module Mandy
   module DSL
-    
-    def self.included(klass)
-      Mandy::Job.default = Mandy::Job.new('Untitled Job')
-    end
-    
-    def set(key, value)
-      Mandy::Job.default.settings[key.to_s] = value
-    end
-    
-    def job_name(name)
-      set "mapred.job.name", name.to_s
-    end
-    
-    def map(&blk)
-      Mandy::Job.default.map(&blk)
-    end
-    
-    def reduce(&blk)
-      Mandy::Job.default.reduce(&blk)
+    def job(name, &blk)
+      job = Mandy::Job.new(name)
+      job.instance_eval(&blk) unless blk.nil?
+      Mandy::Job.jobs << job
+      job
     end
   end
 end

data/lib/job.rb

@@ -1,17 +1,31 @@
 module Mandy
   class Job
     class << self
-      attr_accessor :default
+      def jobs
+        @jobs ||= []
+      end
+      
+      def find_by_name(name)
+        jobs.find {|job| job.name == name }
+      end
     end
     
     attr_reader :settings
+    attr_reader :name
     
     def initialize(name, &blk)
       @name = name
       @settings = {}
+      @mapper_class = Mandy::Mapper
+      @reducer_class = Mandy::Reducer
+      set('mapred.job.name', name)
       instance_eval(&blk) if blk
     end
     
+    def set(key, value)
+      @settings[key.to_s] = value.to_s
+    end
+    
     def map(&blk)
       @mapper_class = Mandy::Mapper.compile(&blk)
     end

data/lib/test_runner.rb

@@ -1,6 +1,6 @@
 module Mandy
   class TestRunner
-    def initialize(job=Mandy::Job.default)
+    def initialize(job=Mandy::Job.jobs.first)
       @job = job
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: trafficbroker-mandy
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Andy Kent