logstash-filter-csv 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2ece5f8729158340a06b5b6cffdc52d04e37282
-  data.tar.gz: d60433bf57b489f80c8a9ed7e627a2d9c5587151
+  metadata.gz: 74d72358f3baf0fb616273e5b0c3d6dac3784aa2
+  data.tar.gz: dc732a6835ae9b9c37a647d9e0541ffdc8f8d130
 SHA512:
-  metadata.gz: ead9e3a32ba1e4fefcda44cef708ff534a88712f11d5d4e813869211fac97058ea5fb3b6ba61207c49c10ae51ee255fa109afd457524d3215076feb045b0e9a0
-  data.tar.gz: 5d59f9e644b5ae002da5962325f16ff54051673e5ea20618f51cf9cccd0dd9b060be5d8ddc59f469d3f507a29ba32f2990615b4b2144389622dd066e25b3cdb8
+  metadata.gz: caea7cbc014ed27d8e207e257558e086e170a20c120551e11cde96cdbe58424b9b2ce7ed7516a7f0292d8b962abad434b4592a9393e23a60640b987e6f22e0d6
+  data.tar.gz: 9d04652e8989ab10a3d5ec11961fcf53a0ee666557065a6f4e6a017399ef9e8b30e34b1302dd06f55f14673cc5614687f050fb6583ee6f496822739a9884ac1a

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 2.1.0
+ - Added support for not parsing coluns without a defined header.
+ - Added support for dropping columns that has no value
+ - Added support for type conversion within the filter
+ - Fix unnecessary source field mutation. Fixes #18
+ - Refactored specs to avoid using sample and insist in favor of rspec3
+   helper methods.
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-csv-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-csv-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/csv.rb

@@ -36,37 +36,49 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
   # Defaults to writing to the root of the event.
   config :target, :validate => :string
 
-  public
-  def register
+  # Define whether column names should autogenerated or not.
+  # Defaults to true. If set to false, columns not having a header specified will not be parsed.
+  config :autogenerate_column_names, :validate => :boolean, :default => true
 
-    # Nothing to do here
+  # Define whether empty columns should be skipped.
+  # Defaults to false. If set to true, columns containing no value will not get set.
+  config :skip_empty_columns, :validate => :boolean, :default => false
 
-  end # def register
+  # Define a set of datatype conversions to be applied to columns.
+  # Possible conversions are integer, float, date, date_time, boolean
+  #
+  # # Example:
+  # [source,ruby]
+  #     filter {
+  #       csv {
+  #         convert => { "column1" => "integer", "column2" => "boolean" }
+  #       }
+  #     }
+  config :convert, :validate => :hash, :default => {}
 
-  public
-  def filter(event)
-    
 
-    @logger.debug("Running csv filter", :event => event)
+  ##
+  # List of valid conversion types used for the convert option
+  ##
+  VALID_CONVERT_TYPES = [ "integer", "float", "date", "date_time", "boolean" ].freeze
 
-    matches = 0
 
-    if event[@source]
-      if event[@source].is_a?(String)
-        event[@source] = [event[@source]]
+  def register
+    # validate conversion types to be the valid ones.
+    @convert.each_pair do |column, type|
+      if !VALID_CONVERT_TYPES.include?(type)
+        raise LogStash::ConfigurationError, "#{type} is not a valid conversion type."
       end
+    end
+  end # def register
 
-      if event[@source].length > 1
-        @logger.warn("csv filter only works on fields of length 1",
-                     :source => @source, :value => event[@source],
-                     :event => event)
-        return
-      end
+  def filter(event)
+    @logger.debug("Running csv filter", :event => event)
 
-      raw = event[@source].first
+    if event[@source]
+      source = event[@source].clone
       begin
-        values = CSV.parse_line(raw, :col_sep => @separator, :quote_char => @quote_char)
-
+        values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)
         if @target.nil?
           # Default is to write to the root of the event.
           dest = event
@@ -75,15 +87,21 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
         end
 
         values.each_index do |i|
-          field_name = @columns[i] || "column#{i+1}"
-          dest[field_name] = values[i]
+          if !(@skip_empty_columns && (values[i].nil? || values[i].empty?))
+            if !ignore_field?(i)
+              field_name       = @columns[i] ? @columns[i] : "column#{i+1}"
+              dest[field_name] = if should_transform?(field_name)
+                                   transform(field_name, values[i])
+                                 else
+                                   values[i]
+                                 end
+            end
+          end
         end
-
         filter_matched(event)
       rescue => e
         event.tag "_csvparsefailure"
-        @logger.warn("Trouble parsing csv", :source => @source, :raw => raw,
-                      :exception => e)
+        @logger.warn("Trouble parsing csv", :field => @source, :source => source, :exception => e)
         return
       end # begin
     end # if event
@@ -92,5 +110,44 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
 
   end # def filter
 
+  private
+
+  def ignore_field?(index)
+    !@columns[index] && !@autogenerate_column_names
+  end
+
+  def should_transform?(field_name)
+    !@convert[field_name].nil?
+  end
+
+  def transform(field_name, value)
+    transformation = @convert[field_name].to_sym
+    converters[transformation].call(value)
+  end
+
+  def converters
+    @converters ||= {
+      :integer => lambda do |value|
+        CSV::Converters[:integer].call(value)
+      end,
+      :float => lambda do |value|
+        CSV::Converters[:float].call(value)
+
+      end,
+      :date => lambda do |value|
+        CSV::Converters[:date].call(value)
+
+      end,
+      :date_time => lambda do |value|
+        CSV::Converters[:date_time].call(value)
+      end,
+      :boolean => lambda do |value|
+         value = value.strip.downcase
+         return false if value == "false"
+         return true  if value == "true"
+         return value
+      end
+    }
+  end
 end # class LogStash::Filters::Csv
 

data/logstash-filter-csv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-csv'
-  s.version         = '2.0.2'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The CSV filter takes an event field containing CSV data, parses it, and stores it as individual fields (can optionally specify the names)."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/csv_spec.rb

@@ -1,175 +1,249 @@
 # encoding: utf-8
-
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/csv"
 
 describe LogStash::Filters::CSV do
 
-  describe "all defaults" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        csv { }
-      }
-    CONFIG
-
-    sample "big,bird,sesame street" do
-      insist { subject["column1"] } == "big"
-      insist { subject["column2"] } == "bird"
-      insist { subject["column3"] } == "sesame street"
-    end
-  end
+  subject(:plugin) { LogStash::Filters::CSV.new(config) }
+  let(:config) { Hash.new }
 
-  describe "custom separator" do
-    config <<-CONFIG
-      filter {
-        csv {
-          separator => ";"
-        }
-      }
-    CONFIG
-
-    sample "big,bird;sesame street" do
-      insist { subject["column1"] } == "big,bird"
-      insist { subject["column2"] } == "sesame street"
-    end
-  end
+  let(:doc)   { "" }
+  let(:event) { LogStash::Event.new("message" => doc) }
 
-  describe "custom quote char" do
-    config <<-CONFIG
-      filter {
-        csv {
-          quote_char => "'"
-        }
-      }
-    CONFIG
-
-    sample "big,bird,'sesame street'" do
-      insist { subject["column1"] } == "big"
-      insist { subject["column2"] } == "bird"
-      insist { subject["column3"] } == "sesame street"
-    end
-  end
+  describe "registration" do
+
+    context "when using invalid data types" do
+      let(:config) do
+        { "convert" => { "custom1" => "integer", "custom3" => "wrong_type" },
+          "columns" => ["custom1", "custom2", "custom3"] }
+      end
 
-  describe "default quote char" do
-    config <<-CONFIG
-      filter {
-        csv {
-        }
-      }
-    CONFIG
-
-    sample 'big,bird,"sesame, street"' do
-      insist { subject["column1"] } == "big"
-      insist { subject["column2"] } == "bird"
-      insist { subject["column3"] } == "sesame, street"
+      it "should register" do
+        input = LogStash::Plugin.lookup("filter", "csv").new(config)
+        expect {input.register}.to raise_error
+      end
     end
   end
-  describe "null quote char" do
-    config <<-CONFIG
-      filter {
-        csv {
-          quote_char => "\x00"
-        }
-      }
-    CONFIG
-
-    sample 'big,bird,"sesame" street' do
-      insist { subject["column1"] } == 'big'
-      insist { subject["column2"] } == 'bird'
-      insist { subject["column3"] } == '"sesame" street'
+
+  describe "receive" do
+
+    before(:each) do
+      plugin.register
     end
-  end
 
-  describe "given columns" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        csv {
-          columns => ["first", "last", "address" ]
-        }
-      }
-    CONFIG
-
-    sample "big,bird,sesame street" do
-      insist { subject["first"] } == "big"
-      insist { subject["last"] } == "bird"
-      insist { subject["address"] } == "sesame street"
+    describe "all defaults" do
+
+      let(:config) { Hash.new }
+
+      let(:doc) { "big,bird,sesame street" }
+
+      it "extract all the values" do
+        plugin.filter(event)
+        expect(event["column1"]).to eq("big")
+        expect(event["column2"]).to eq("bird")
+        expect(event["column3"]).to eq("sesame street")
+      end
+
+      it "should not mutate the source field" do
+        plugin.filter(event)
+        expect(event["message"]).to be_kind_of(String)
+      end
     end
-  end
 
-  describe "parse csv with more data than defined column names" do
-    config <<-CONFIG
-      filter {
-        csv {
-          columns => ["custom1", "custom2"]
-        }
-      }
-    CONFIG
-
-    sample "val1,val2,val3" do
-      insist { subject["custom1"] } == "val1"
-      insist { subject["custom2"] } == "val2"
-      insist { subject["column3"] } == "val3"
+    describe "custom separator" do
+      let(:doc) { "big,bird;sesame street" }
+
+      let(:config) do
+        { "separator" => ";" }
+      end
+      it "extract all the values" do
+        plugin.filter(event)
+        expect(event["column1"]).to eq("big,bird")
+        expect(event["column2"]).to eq("sesame street")
+      end
     end
-  end
 
+    describe "quote char" do
+      let(:doc) { "big,bird,'sesame street'" }
+
+      let(:config) do
+        { "quote_char" => "'"}
+      end
+
+      it "extract all the values" do
+        plugin.filter(event)
+        expect(event["column1"]).to eq("big")
+        expect(event["column2"]).to eq("bird")
+        expect(event["column3"]).to eq("sesame street")
+      end
+
+      context "using the default one" do
+        let(:doc) { 'big,bird,"sesame, street"' }
+        let(:config) { Hash.new }
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["column1"]).to eq("big")
+          expect(event["column2"]).to eq("bird")
+          expect(event["column3"]).to eq("sesame, street")
+        end
+      end
+
+      context "using a null" do
+        let(:doc) { 'big,bird,"sesame" street' }
+        let(:config) do
+          { "quote_char" => "\x00" }
+        end
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["column1"]).to eq("big")
+          expect(event["column2"]).to eq("bird")
+          expect(event["column3"]).to eq('"sesame" street')
+        end
+      end
+    end
 
-  describe "parse csv from a given source with column names" do
-    config <<-CONFIG
-      filter {
-        csv {
-          source => "datafield"
-          columns => ["custom1", "custom2", "custom3"]
-        }
-      }
-    CONFIG
-
-    sample("datafield" => "val1,val2,val3") do
-      insist { subject["custom1"] } == "val1"
-      insist { subject["custom2"] } == "val2"
-      insist { subject["custom3"] } == "val3"
+    describe "given column names" do
+      let(:doc)    { "big,bird,sesame street" }
+      let(:config) do
+        { "columns" => ["first", "last", "address" ] }
+      end
+
+      it "extract all the values" do
+        plugin.filter(event)
+        expect(event["first"]).to eq("big")
+        expect(event["last"]).to eq("bird")
+        expect(event["address"]).to eq("sesame street")
+      end
+
+      context "parse csv without autogeneration of names" do
+
+        let(:doc)    { "val1,val2,val3" }
+        let(:config) do
+          {  "autogenerate_column_names" => false,
+             "columns" => ["custom1", "custom2"] }
+        end
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["custom1"]).to eq("val1")
+          expect(event["custom2"]).to eq("val2")
+          expect(event["column3"]).to be_falsey
+        end
+      end
+
+      context "parse csv skipping empty columns" do
+
+        let(:doc)    { "val1,,val3" }
+
+        let(:config) do
+          { "skip_empty_columns" => true,
+            "source" => "datafield",
+            "columns" => ["custom1", "custom2", "custom3"] }
+        end
+
+        let(:event) { LogStash::Event.new("datafield" => doc) }
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["custom1"]).to eq("val1")
+          expect(event["custom2"]).to be_falsey
+          expect(event["custom3"]).to eq("val3")
+        end
+      end
+
+      context "parse csv with more data than defined" do
+        let(:doc)    { "val1,val2,val3" }
+        let(:config) do
+          { "columns" => ["custom1", "custom2"] }
+        end
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["custom1"]).to eq("val1")
+          expect(event["custom2"]).to eq("val2")
+          expect(event["column3"]).to eq("val3")
+        end
+      end
+
+      context "parse csv from a given source" do
+        let(:doc)    { "val1,val2,val3" }
+        let(:config) do
+          { "source"  => "datafield",
+            "columns" => ["custom1", "custom2", "custom3"] }
+        end
+        let(:event) { LogStash::Event.new("datafield" => doc) }
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["custom1"]).to eq("val1")
+          expect(event["custom2"]).to eq("val2")
+          expect(event["custom3"]).to eq("val3")
+        end
+      end
     end
-  end
 
-  describe "given target" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        csv {
-          target => "data"
-        }
-      }
-    CONFIG
-
-    sample "big,bird,sesame street" do
-      insist { subject["data"]["column1"] } == "big"
-      insist { subject["data"]["column2"] } == "bird"
-      insist { subject["data"]["column3"] } == "sesame street"
+    describe "givin target" do
+      let(:config) do
+        { "target" => "data" }
+      end
+      let(:doc)   { "big,bird,sesame street" }
+      let(:event) { LogStash::Event.new("message" => doc) }
+
+      it "extract all the values" do
+        plugin.filter(event)
+        expect(event["data"]["column1"]).to eq("big")
+        expect(event["data"]["column2"]).to eq("bird")
+        expect(event["data"]["column3"]).to eq("sesame street")
+      end
+
+      context "when having also source" do
+        let(:config) do
+          {  "source" => "datain",
+             "target" => "data" }
+        end
+        let(:event) { LogStash::Event.new("datain" => doc) }
+        let(:doc)   { "big,bird,sesame street" }
+
+        it "extract all the values" do
+          plugin.filter(event)
+          expect(event["data"]["column1"]).to eq("big")
+          expect(event["data"]["column2"]).to eq("bird")
+          expect(event["data"]["column3"]).to eq("sesame street")
+        end
+      end
     end
-  end
 
-  describe "given target and source" do
-    # The logstash config goes here.
-    # At this time, only filters are supported.
-    config <<-CONFIG
-      filter {
-        csv {
-          source => "datain"
-          target => "data"
-        }
-      }
-    CONFIG
-
-    sample("datain" => "big,bird,sesame street") do
-      insist { subject["data"]["column1"] } == "big"
-      insist { subject["data"]["column2"] } == "bird"
-      insist { subject["data"]["column3"] } == "sesame street"
+    describe "using field convertion" do
+
+      let(:config) do
+        { "convert" => { "column1" => "integer", "column3" => "boolean" } }
+      end
+      let(:doc)   { "1234,bird,false" }
+      let(:event) { LogStash::Event.new("message" => doc) }
+
+      it "get converted values to the expected type" do
+        plugin.filter(event)
+        expect(event["column1"]).to eq(1234)
+        expect(event["column2"]).to eq("bird")
+        expect(event["column3"]).to eq(false)
+      end
+
+      context "when using column names" do
+
+        let(:config) do
+          { "convert" => { "custom1" => "integer", "custom3" => "boolean" },
+            "columns" => ["custom1", "custom2", "custom3"] }
+        end
+
+        it "get converted values to the expected type" do
+          plugin.filter(event)
+          expect(event["custom1"]).to eq(1234)
+          expect(event["custom2"]).to eq("bird")
+          expect(event["custom3"]).to eq(false)
+        end
+      end
     end
   end
-
-
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-csv
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,20 +28,22 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []