wukong 2.0.1 → 2.0.2

data/bin/wu-lign

@@ -98,8 +98,8 @@ FORMAT_GUESSING_LINES = 500
 # widest column to set
 MAX_MAX_WIDTH = 100
 
-INT_RE   = /\A\d+\z/
-FLOAT_RE = /\A(\d+)(?:\.(\d+))?(?:e-?\d+)?\z/
+INT_RE   = /\A[\d,]+\z/
+FLOAT_RE = /\A([\d,]+)(?:\.(\d+))?(?:e-?\d+)?\z/
 
 def get_type val
   case
@@ -161,7 +161,7 @@ format = maxw.zip(col_types, col_minmag, col_maxmag, ARGV).map do |width, type,
   case type
   when :mixed, nil then lambda{|s| "%-#{width}s" % s }
   when :str        then lambda{|s| "%-#{width}s" % s }
-  when :int        then lambda{|s| "%#{width}d"  % s.to_i }
+  when :int        then lambda{|s| "%#{width}d"  % s.gsub(/\D+/, "").to_i }
   when :float      then lambda{|s| "%#{maxmag+minmag+2}.#{minmag}f" % s.to_f }
   else raise "oops type #{type}"  end
 end

data/lib/wukong.rb

@@ -10,6 +10,7 @@ module Wukong
   autoload :Streamer,        'wukong/streamer'
   autoload :Store,           'wukong/store'
   autoload :FilenamePattern, 'wukong/filename_pattern'
+  autoload :Decorator,       'wukong/decorator'
 
   def self.run mapper, reducer=nil, options={}
     Wukong::Script.new(mapper, reducer, options).run

data/lib/wukong/decorator.rb

@@ -0,0 +1,28 @@
+require 'java'
+
+java_import 'com.cloudera.flume.core.Event'
+java_import 'com.cloudera.flume.core.EventImpl'
+java_import 'com.cloudera.flume.core.EventSinkDecorator'
+
+module Wukong 
+  class Decorator < EventSinkDecorator
+
+    def initialize(mapper, reducer=nil, options={})
+      super(nil)
+      @mapper = mapper.new
+    end
+
+    def append(e)
+      line   = String.from_java_bytes(e.getBody)
+      record = @mapper.recordize(line.chomp)
+      @mapper.process(*record) do |output|
+        processed = output.to_flat.join("\t")
+        event     = EventImpl.new(processed.to_java_bytes, e.getTimestamp, e.getPriority, e.getNanos, e.getHost, e.getAttrs)
+        super event
+      end
+    end
+
+    def run() self ; end
+
+  end
+end

data/lib/wukong/encoding.rb

@@ -68,7 +68,7 @@ module Wukong
   #
   def self.decode_str str, strategy=:xml
     case strategy
-    when :xml        then HTMLEntities.decode_entities(str)
+    when :xml        then self.html_encoder.decode(str)
     when :url        then Addressable::URI.unencode_component(str)
     else raise "Don't know how to decode with strategy #{strategy}"
     end

data/wukong.gemspec

@@ -4,21 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{wukong}
-  s.version = "2.0.1"
+  s.name = "wukong"
+  s.version = "2.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2011-07-01}
-  s.description = %q{  Treat your dataset like a:
-
-      * stream of lines when it's efficient to process by lines
-      * stream of field arrays when it's efficient to deal directly with fields
-      * stream of lightweight objects when it's efficient to deal with objects
-
-  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.
-}
-  s.email = %q{flip@infochimps.org}
+  s.date = "2011-11-04"
+  s.description = "  Treat your dataset like a:\n\n      * stream of lines when it's efficient to process by lines\n      * stream of field arrays when it's efficient to deal directly with fields\n      * stream of lightweight objects when it's efficient to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.\n"
+  s.email = "flip@infochimps.org"
   s.executables = ["hdp-du", "hdp-sync", "hdp-wc", "wu-lign", "wu-sum", "hdp-parts_to_keys.rb"]
   s.extra_rdoc_files = [
     "LICENSE.textile",
@@ -180,6 +173,7 @@ Gem::Specification.new do |s|
     "lib/wukong/datatypes.rb",
     "lib/wukong/datatypes/enum.rb",
     "lib/wukong/datatypes/fake_types.rb",
+    "lib/wukong/decorator.rb",
     "lib/wukong/encoding.rb",
     "lib/wukong/encoding/asciize.rb",
     "lib/wukong/extensions.rb",
@@ -254,46 +248,10 @@ Gem::Specification.new do |s|
     "spec/wukong/script_spec.rb",
     "wukong.gemspec"
   ]
-  s.homepage = %q{http://mrflip.github.com/wukong}
+  s.homepage = "http://mrflip.github.com/wukong"
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.5.0}
-  s.summary = %q{Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.}
-  s.test_files = [
-    "examples/contrib/jeans/normalize.rb",
-    "examples/contrib/jeans/sizes.rb",
-    "examples/corpus/bucket_counter.rb",
-    "examples/corpus/dbpedia_abstract_to_sentences.rb",
-    "examples/corpus/sentence_bigrams.rb",
-    "examples/corpus/sentence_coocurrence.rb",
-    "examples/corpus/words_to_bigrams.rb",
-    "examples/emr/elastic_mapreduce_example.rb",
-    "examples/ignore_me/counting.rb",
-    "examples/ignore_me/grouper.rb",
-    "examples/network_graph/adjacency_list.rb",
-    "examples/network_graph/breadth_first_search.rb",
-    "examples/network_graph/gen_2paths.rb",
-    "examples/network_graph/gen_multi_edge.rb",
-    "examples/network_graph/gen_symmetric_links.rb",
-    "examples/pagerank/pagerank.rb",
-    "examples/pagerank/pagerank_initialize.rb",
-    "examples/sample_records.rb",
-    "examples/server_logs/apache_log_parser.rb",
-    "examples/server_logs/breadcrumbs.rb",
-    "examples/server_logs/logline.rb",
-    "examples/server_logs/nook.rb",
-    "examples/server_logs/nook/faraday_dummy_adapter.rb",
-    "examples/server_logs/user_agent.rb",
-    "examples/simple_word_count.rb",
-    "examples/size.rb",
-    "examples/stats/avg_value_frequency.rb",
-    "examples/stats/binning_percentile_estimator.rb",
-    "examples/stats/rank_and_bin.rb",
-    "examples/stupidly_simple_filter.rb",
-    "examples/word_count.rb",
-    "spec/spec_helper.rb",
-    "spec/wukong/encoding_spec.rb",
-    "spec/wukong/script_spec.rb"
-  ]
+  s.rubygems_version = "1.8.10"
+  s.summary = "Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease."
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

metadata

@@ -1,87 +1,89 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: wukong
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 2.0.2
   prerelease: 
-  version: 2.0.1
 platform: ruby
-authors: 
+authors:
 - Philip (flip) Kromer
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-07-01 00:00:00 -05:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-11-04 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rspec
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2152107000 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.2.9
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: yard
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2152107000
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &2152105540 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: addressable
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2152105540
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: &2152104640 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: extlib
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2152104640
+- !ruby/object:Gem::Dependency
+  name: extlib
+  requirement: &2152103820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: htmlentities
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *2152103820
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: &2152102940 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: configliere
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: *2152102940
+- !ruby/object:Gem::Dependency
+  name: configliere
+  requirement: &2152101420 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id006
-description: "  Treat your dataset like a:\n\n      * stream of lines when it's efficient to process by lines\n      * stream of field arrays when it's efficient to deal directly with fields\n      * stream of lightweight objects when it's efficient to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query language, and the cat on your command line.\n"
+  prerelease: false
+  version_requirements: *2152101420
+description: ! "  Treat your dataset like a:\n\n      * stream of lines when it's
+  efficient to process by lines\n      * stream of field arrays when it's efficient
+  to deal directly with fields\n      * stream of lightweight objects when it's efficient
+  to deal with objects\n\n  Wukong is friends with Hadoop the elephant, Pig the query
+  language, and the cat on your command line.\n"
 email: flip@infochimps.org
-executables: 
+executables:
 - hdp-du
 - hdp-sync
 - hdp-wc
@@ -89,11 +91,10 @@ executables:
 - wu-sum
 - hdp-parts_to_keys.rb
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - LICENSE.textile
 - README.textile
-files: 
+files:
 - CHANGELOG.textile
 - INSTALL.textile
 - LICENSE.textile
@@ -249,6 +250,7 @@ files:
 - lib/wukong/datatypes.rb
 - lib/wukong/datatypes/enum.rb
 - lib/wukong/datatypes/fake_types.rb
+- lib/wukong/decorator.rb
 - lib/wukong/encoding.rb
 - lib/wukong/encoding/asciize.rb
 - lib/wukong/extensions.rb
@@ -322,66 +324,29 @@ files:
 - spec/wukong/encoding_spec.rb
 - spec/wukong/script_spec.rb
 - wukong.gemspec
-has_rdoc: true
 homepage: http://mrflip.github.com/wukong
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.0
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
-summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use it, yet handles terabyte-scale computation with ease.
-test_files: 
-- examples/contrib/jeans/normalize.rb
-- examples/contrib/jeans/sizes.rb
-- examples/corpus/bucket_counter.rb
-- examples/corpus/dbpedia_abstract_to_sentences.rb
-- examples/corpus/sentence_bigrams.rb
-- examples/corpus/sentence_coocurrence.rb
-- examples/corpus/words_to_bigrams.rb
-- examples/emr/elastic_mapreduce_example.rb
-- examples/ignore_me/counting.rb
-- examples/ignore_me/grouper.rb
-- examples/network_graph/adjacency_list.rb
-- examples/network_graph/breadth_first_search.rb
-- examples/network_graph/gen_2paths.rb
-- examples/network_graph/gen_multi_edge.rb
-- examples/network_graph/gen_symmetric_links.rb
-- examples/pagerank/pagerank.rb
-- examples/pagerank/pagerank_initialize.rb
-- examples/sample_records.rb
-- examples/server_logs/apache_log_parser.rb
-- examples/server_logs/breadcrumbs.rb
-- examples/server_logs/logline.rb
-- examples/server_logs/nook.rb
-- examples/server_logs/nook/faraday_dummy_adapter.rb
-- examples/server_logs/user_agent.rb
-- examples/simple_word_count.rb
-- examples/size.rb
-- examples/stats/avg_value_frequency.rb
-- examples/stats/binning_percentile_estimator.rb
-- examples/stats/rank_and_bin.rb
-- examples/stupidly_simple_filter.rb
-- examples/word_count.rb
-- spec/spec_helper.rb
-- spec/wukong/encoding_spec.rb
-- spec/wukong/script_spec.rb
+summary: Hadoop Streaming for Ruby. Wukong makes Hadoop so easy a chimpanzee can use
+  it, yet handles terabyte-scale computation with ease.
+test_files: []