logstash-filter-csv 2.1.0 → 2.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 74d72358f3baf0fb616273e5b0c3d6dac3784aa2
-  data.tar.gz: dc732a6835ae9b9c37a647d9e0541ffdc8f8d130
+  metadata.gz: 5f4aa9f9214e37010bf9e1248710142f2c7cb8af
+  data.tar.gz: 0c56b56dcd712437cbf0922a1f26a5cd028ee792
 SHA512:
-  metadata.gz: caea7cbc014ed27d8e207e257558e086e170a20c120551e11cde96cdbe58424b9b2ce7ed7516a7f0292d8b962abad434b4592a9393e23a60640b987e6f22e0d6
-  data.tar.gz: 9d04652e8989ab10a3d5ec11961fcf53a0ee666557065a6f4e6a017399ef9e8b30e34b1302dd06f55f14673cc5614687f050fb6583ee6f496822739a9884ac1a
+  metadata.gz: f6ac80e0d09aabff2022a9096866a6890f9d1a44ba24aa99386c43f4dc9173f4524281c5d5f5fc1ab9f2d4fd2d905a36a43b094239dad932baf014a4529a3ce6
+  data.tar.gz: 481be1574ce4fcef426cf259aa7ed09a5d3a66d8f9016a629b475a640a4e4e9beb91a0bc9621bfe7f33d4c6a91f34345dbb03bdcc9bd0a8866b86913872a0e75

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.1.1
+ - Fixed field references, refactors converters, code cleanups
+
 ## 2.1.0
  - Added support for not parsing coluns without a defined header.
  - Added support for dropping columns that has no value
@@ -7,7 +10,7 @@
    helper methods.
 
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/lib/logstash/filters/csv.rb

@@ -56,98 +56,89 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
   #     }
   config :convert, :validate => :hash, :default => {}
 
-
-  ##
-  # List of valid conversion types used for the convert option
-  ##
-  VALID_CONVERT_TYPES = [ "integer", "float", "date", "date_time", "boolean" ].freeze
-
+  CONVERTERS = {
+    :integer => lambda do |value|
+      CSV::Converters[:integer].call(value)
+    end,
+
+    :float => lambda do |value|
+      CSV::Converters[:float].call(value)
+    end,
+
+    :date => lambda do |value|
+      CSV::Converters[:date].call(value)
+    end,
+
+    :date_time => lambda do |value|
+      CSV::Converters[:date_time].call(value)
+    end,
+
+    :boolean => lambda do |value|
+       value = value.strip.downcase
+       return false if value == "false"
+       return true  if value == "true"
+       return value
+    end
+  }
+  CONVERTERS.default = lambda {|v| v}
+  CONVERTERS.freeze
 
   def register
     # validate conversion types to be the valid ones.
-    @convert.each_pair do |column, type|
-      if !VALID_CONVERT_TYPES.include?(type)
-        raise LogStash::ConfigurationError, "#{type} is not a valid conversion type."
-      end
-    end
-  end # def register
+    bad_types = @convert.values.select do |type|
+      !CONVERTERS.has_key?(type.to_sym)
+    end.uniq
+
+    raise(LogStash::ConfigurationError, "Invalid conversion types: #{bad_types.join(', ')}") unless bad_types.empty?
+
+    # @convert_symbols contains the symbolized types to avoid symbol conversion in the transform method
+    @convert_symbols = @convert.inject({}){|result, (k, v)| result[k] = v.to_sym; result}
+
+    # make sure @target is in the format [field name] if defined, i.e. surrounded by brakets
+    @target = "[#{@target}]" if @target && @target !~ /^\[[^\[\]]+\]$/
+  end
 
   def filter(event)
-    @logger.debug("Running csv filter", :event => event)
+    @logger.debug? && @logger.debug("Running csv filter", :event => event)
 
-    if event[@source]
-      source = event[@source].clone
+    if (source = event[@source])
       begin
         values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)
-        if @target.nil?
-          # Default is to write to the root of the event.
-          dest = event
-        else
-          dest = event[@target] ||= {}
-        end
 
         values.each_index do |i|
-          if !(@skip_empty_columns && (values[i].nil? || values[i].empty?))
-            if !ignore_field?(i)
-              field_name       = @columns[i] ? @columns[i] : "column#{i+1}"
-              dest[field_name] = if should_transform?(field_name)
-                                   transform(field_name, values[i])
-                                 else
-                                   values[i]
-                                 end
+          unless (@skip_empty_columns && (values[i].nil? || values[i].empty?))
+            unless ignore_field?(i)
+              field_name = @columns[i] || "column#{i + 1}"
+              event[field_ref(field_name)] = transform(field_name, values[i])
             end
           end
         end
+
         filter_matched(event)
       rescue => e
-        event.tag "_csvparsefailure"
-        @logger.warn("Trouble parsing csv", :field => @source, :source => source, :exception => e)
+        event.tag("_csvparsefailure")
+        @logger.warn("Error parsing csv", :field => @source, :source => source, :exception => e)
         return
-      end # begin
-    end # if event
-
-    @logger.debug("Event after csv filter", :event => event)
+      end
+    end
 
-  end # def filter
+    @logger.debug? && @logger.debug("Event after csv filter", :event => event)
+  end
 
   private
 
-  def ignore_field?(index)
-    !@columns[index] && !@autogenerate_column_names
+  # construct the correct Event field reference for given field_name, taking into account @target
+  # @param field_name [String] the bare field name without brakets
+  # @return [String] fully qualified Event field reference also taking into account @target prefix
+  def field_ref(field_name)
+    "#{@target}[#{field_name}]"
   end
 
-  def should_transform?(field_name)
-    !@convert[field_name].nil?
+  def ignore_field?(index)
+    !@columns[index] && !@autogenerate_column_names
   end
 
   def transform(field_name, value)
-    transformation = @convert[field_name].to_sym
-    converters[transformation].call(value)
+    CONVERTERS[@convert_symbols[field_name]].call(value)
   end
-
-  def converters
-    @converters ||= {
-      :integer => lambda do |value|
-        CSV::Converters[:integer].call(value)
-      end,
-      :float => lambda do |value|
-        CSV::Converters[:float].call(value)
-
-      end,
-      :date => lambda do |value|
-        CSV::Converters[:date].call(value)
-
-      end,
-      :date_time => lambda do |value|
-        CSV::Converters[:date_time].call(value)
-      end,
-      :boolean => lambda do |value|
-         value = value.strip.downcase
-         return false if value == "false"
-         return true  if value == "true"
-         return value
-      end
-    }
-  end
-end # class LogStash::Filters::Csv
-
+end

data/logstash-filter-csv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-csv'
-  s.version         = '2.1.0'
+  s.version         = '2.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The CSV filter takes an event field containing CSV data, parses it, and stores it as individual fields (can optionally specify the names)."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-csv
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-01 00:00:00.000000000 Z
+date: 2015-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core