hivemeta 0.0.6 → 0.1.0

data/CHANGELOG

@@ -1,3 +1,7 @@
+* 2011-05-21 - fsf
+- new: added table.process* opts hash
+- new: added :ignore_field_count, :field_count_warning, and :file opts
+
 * 2011-05-19 - fsf
 - thank you ruby-prof!
 - perf: 4x+ faster ... now basically on par with manual split into array

data/lib/hivemeta/record.rb

@@ -3,17 +3,13 @@ module HiveMeta
   class FieldCountError < StandardError ; end
 
   class Record
-    def initialize(line, table)
+    def initialize(line, table, opts = {})
       @fields = line.chomp.split(table.delimiter, -1)
       if @fields.size != table.columns.size
-        raise FieldCountError
+        raise FieldCountError if not opts[:ignore_field_count]
       end
 
       @table = table
-      #@columns = {}
-#      table.each_col_with_index do |col_name, i|
-#        #@columns[col_name.to_sym] = @fields[i]
-#      end
     end
 
     # allow for column access via column name as an index
@@ -23,7 +19,6 @@ module HiveMeta
     # example: rec[7]
     def [] index
       return "#{@fields[index]}" if index.is_a? Integer
-      #"#{@columns[index.to_sym]}"
       "#{@fields[@table.indexes[index.to_sym]]}"
     end
 
@@ -31,7 +26,6 @@ module HiveMeta
     # example: rec.col_name
     def method_missing(id, *args)
       return @fields[@table.indexes[id]] if @fields[@table.indexes[id]]
-      #return @columns[id] if @columns[id]
       raise NoMethodError
     end
   end

data/lib/hivemeta/table.rb

@@ -41,27 +41,30 @@ module HiveMeta
 
     # process a row and return a record that can be queried
     # by column name in a variety of ways
-    def process_row(line)
+    def process_row(line, opts = {})
       return nil if not line
       if block_given?
-        yield Record.new(line, self)
+        yield Record.new(line, self, opts)
       else
-        return Record.new(line, self)
+        return Record.new(line, self, opts)
       end
     end
 
     # process all input (default to STDIN for Hadoop Streaming)
     # via a provided block
-    def process(f = STDIN, warning = nil)
+    def process(opts = {})
+      f = opts[:file] || STDIN
+
       if not block_given?
-        return process_row f.readline
+        return process_row(f.readline, opts)
       end
 
       f.each_line do |line|
         begin
-          process_row(line) {|row| yield row}
+          process_row(line, opts) {|row| yield row}
         rescue HiveMeta::FieldCountError
-          warning ||= "reporter:counter:bad_data,row_size,1"
+          warning = opts[:field_count_warning]
+          warning ||= "reporter:counter:HiveMeta,FieldCountError,1"
           STDERR.puts warning
           next
         end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
+  - 1
   - 0
-  - 6
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors: 
 - Frank Fejes
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-19 00:00:00 -05:00
+date: 2011-05-21 00:00:00 -05:00
 default_executable: 
 dependencies: []