humboldt 1.1.0-java → 1.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1b68b1b122d8e69538a1ed888cd8ede41e07a4a7
-  data.tar.gz: b658d3c93c062116ecec9c1b1f3b09fb26b5ff49
+  metadata.gz: 6a97f4f665fc9cd31b2243197be7f0486ce1d078
+  data.tar.gz: 5cc3932bfdba9f35c13db5f9ecbc566129887ba9
 SHA512:
-  metadata.gz: 17780975cfc99cb6f4f4639d23dc3c3336085c5343e37b37841eb7d8eacfc49d7963d5f217d90f9495f4ed8aed1ac36e45419e79d7dddc1cce93aa87b08621c6
-  data.tar.gz: 8b79f3d82f416d5180ff657ca01de80b6370b745a6587aec44efcdfb4918196d0d0b61c84f46ca3fbb4ca0d10e249c5cbc1cf64e241aaa7b3c47771ef76dd43c
+  metadata.gz: c66f66dc44da611770ebeec44e3c1ecb16e17207fee63db3ae0cc9d36704e860fead33cf1dbefe5f8dce4c628d08470ee8ec330c2e8468ae455b35100c1ae8ba
+  data.tar.gz: 20306297025d1bc59a034c9e8f166ff30e682ccca731e8a36df2e2b785645bbe0c227858c99cee69ed0dad67cf3dc615107f0f5dfdb3d561ba64b41df50687f0

data/lib/ext/rubydoop.rb

@@ -109,7 +109,7 @@ module Rubydoop
     # for example when you don't know how wide the part of the key that you want
     # use is. In the example above if you use the domain to identify sites these
     # can be of different length. If your visitor IDs are 20 characters you can
-    # use 0 and -20 as your indexes.
+    # use 0 and -21 as your indexes.
     #
     # @param [Fixnum] start_index The first index of the slice, negative numbers
     #   are counted from the end

data/lib/humboldt/cli.rb

@@ -2,7 +2,7 @@
 
 require 'thor'
 require 'aws'
-require 'open3'
+require 'rubydoop/version'
 require 'rubydoop/package' # this prints an annoying warning in JRuby 1.7.0.RC1
 require 'humboldt/emr_flow'
 require 'humboldt/hadoop_status_filter'
@@ -204,23 +204,24 @@ module Humboldt
 
     def run_command(*args)
       say_status(:running, 'Hadoop started')
-      Open3.popen3(*args) do |stdin, stdout, stderr, wait_thr|
-        stdin.close
-        stdout_printer = Thread.new(stdout) do |stdout|
-          while line = stdout.gets
-            say(line.chomp)
+      HadoopStatusFilter.run_command_with_filtering(*args) do |type, *args|
+        case type
+        when :stderr
+          unless options.silent?
+            say(args.first, :red)
           end
-        end
-        stderr_printer = Thread.new(stderr) do |stderr|
-          filter = HadoopStatusFilter.new(stderr, self, options.silent?)
-          filter.run
-        end
-        stdout_printer.join
-        stderr_printer.join
-        if wait_thr.value.exitstatus == 0
-          say_status(:done, 'Job completed')
+        when :status
+          say(args[0], args[1] == :error ? :red : :yellow)
+        when :counters
+          say
+          print_table(args.first)
+          say
+        when :done
+          say('Job completed')
+        when :failed
+          say('Job failed', :red)
         else
-          say_status(:failed, 'Job failed', :red)
+          say(args.first)
         end
       end
     end

data/lib/humboldt/hadoop_status_filter.rb

@@ -1,14 +1,38 @@
 # encoding: utf-8
 
+require 'open3'
+
 module Humboldt
+  # @private
   class HadoopStatusFilter
-    def initialize(hadoop_stderr, shell, silent)
+    def initialize(hadoop_stderr, listener)
       @hadoop_stderr = hadoop_stderr
-      @shell = shell
-      @silent = silent
+      @listener = listener
       @counters = {}
     end
 
+    def self.run_command_with_filtering(*args, &listener)
+      Open3.popen3(*args) do |stdin, stdout, stderr, wait_thr|
+        stdin.close
+        stdout_printer = Thread.new(stdout) do |stdout|
+          while line = stdout.gets
+            listener.call(:stdout, line.chomp)
+          end
+        end
+        stderr_printer = Thread.new(stderr) do |stderr|
+          filter = new(stderr, listener)
+          filter.run
+        end
+        stdout_printer.join
+        stderr_printer.join
+        if wait_thr.value.exitstatus == 0
+          listener.call(:done)
+        else
+          listener.call(:failed)
+        end
+      end
+    end
+
     def run
       counter_group = nil
       while line = @hadoop_stderr.gets
@@ -46,7 +70,7 @@ module Humboldt
             end
           end
         end
-        @shell.say(line.chomp, :red) unless @silent
+        @listener.call(:stderr, line.chomp)
       end
       print_counters_table
     end
@@ -54,7 +78,7 @@ module Humboldt
     private
 
     def hadoop_log?(line)
-      line =~ /(?:INFO|WARN) (?:mapred|input|output|util|jvm|mapreduce)\./
+      line =~ /(?:INFO|WARN) (?:mapred|input|output|util|jvm|mapreduce|compress|reduce)\./
     end
 
     def ignore?(line)
@@ -73,11 +97,11 @@ module Humboldt
     end
 
     def report_progress(map, reduce)
-      @shell.say_status(:progress, "map #{map}%, reduce #{reduce}%")
+      @listener.call(:progress, "map #{map}%, reduce #{reduce}%")
     end
 
     def report_error(line)
-      @shell.say_status(@error_type, line.chomp, @error_type == :error ? :red : :yellow)
+      @listener.call(:status, line.chomp, @error_type)
     end
 
     def print_counters_table
@@ -89,9 +113,7 @@ module Humboldt
         ]
       end
       table.pop
-      @shell.say
-      @shell.print_table(table)
-      @shell.say
+      @listener.call(:counters, table)
     end
   end
 end

data/lib/humboldt/rspec.rb

@@ -83,7 +83,7 @@ module RunnerHelpers
 
     def configuration
       @configuration ||= ::Hadoop::Conf::Configuration.new.tap do |config|
-        config.set 'mapred.job.tracker', 'local'
+        config.set('mapreduce.framework.name', 'local')
       end
     end
 

data/lib/humboldt/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Humboldt
-  VERSION = '1.1.0'.freeze
+  VERSION = '1.1.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: humboldt
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: java
 authors:
 - The Burt Platform Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-01 00:00:00.000000000 Z
+date: 2015-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement