logstash-filter-csv 3.0.7 → 3.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59b2d82198544486d18c6455f6e8825a2eae76902f1794a5c13eba048f4fa7cf
-  data.tar.gz: 48c02801a14ef09f1f68e9fcafb1f8b5715cf621db709aa06debf135fb40834a
+  metadata.gz: 77c9af10a455ef4116b2abb9d75a9a56cc2ccbdec5e5e99ee9ac629e60f9e43a
+  data.tar.gz: fbb3fef988e0e0871cf97eaf30ff72cd1f53f81ce7a266ab769a4377c6614cae
 SHA512:
-  metadata.gz: 4cb6f32efef8c44af5f0bf99e0641a51dcf2647108e35d30498748cdc9c5232c6522048d3133ba18619c2c6d10e61372ad8e1b900fcee93aa063bfc32584dabf
-  data.tar.gz: 75e0e02d3dc794d6d31d326036dee0d35a06b74d058e28c70be3d287e87f932171772f2b93d4ce7bc8eb17107afe02be263d1334c19c54c987786414f6f13fe3
+  metadata.gz: 11514496263cd97eed74e1cac97b3f8beec06ad52eea1539d10a989b7c14a1436ec2b2421baaa6d5bc3461bb745231c606bdb76f21d24728e0685c9d556f7491
+  data.tar.gz: a4a92a82de66f3f55b6782dd04265d4dd13ffb2d1014c41c553ef7faea9eb8b240f8a47a3b3830489adbb7f39d67fa99f3966873de5e30589572691e7d410f2d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.0.8
+  - feature: Added support for tagging empty rows which users can reference to conditionally drop events
+
 ## 3.0.7
   - Update gemspec summary
 

data/CONTRIBUTORS

@@ -12,6 +12,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
+* Abdul Haseeb Hussain (AbdulHaseebHussain)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -39,6 +39,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-quote_char>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-separator>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-skip_empty_columns>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-skip_empty_rows>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-skip_header>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
@@ -129,6 +131,31 @@ Optional.
 Define whether empty columns should be skipped.
 Defaults to false. If set to true, columns containing no value will not get set.
 
+[id="plugins-{type}s-{plugin}-skip_empty_rows"]
+===== `skip_empty_rows` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Define whether empty rows could potentially be skipped.
+Defaults to false. If set to true, rows containing no value will be tagged with "_csvskippedemptyfield".
+This tag can referenced by users if they wish to cancel events using an 'if' conditional statement.
+
+[id="plugins-{type}s-{plugin}-skip_header"]
+===== `skip_header`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Define whether the header should be skipped.
+Defaults to false. If set to true, the header will be skipped.
+Assumes that header is not repeated within further rows as such rows will also be skipped.
+If skip_header is set without autodetect_column_names being set then columns should be set which
+will result in the skipping of any row that exactly matches the specified column values.
+If skip_header and autodetect_column_names are specified then columns should not be specified, in this case
+autodetect_column_names will fill the columns setting in the background, from the first event seen, and any
+subsequent values that match what was autodetected will be skipped.
+
 [id="plugins-{type}s-{plugin}-source"]
 ===== `source` 
 

data/lib/logstash/filters/csv.rb

@@ -41,10 +41,19 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
   # Defaults to true. If set to false, columns not having a header specified will not be parsed.
   config :autogenerate_column_names, :validate => :boolean, :default => true
 
+  # Define whether the header should be skipped or not
+  # Defaults to false, If set to true, the header is dropped
+  config :skip_header, :validate => :boolean, :default => false
+
   # Define whether empty columns should be skipped.
   # Defaults to false. If set to true, columns containing no value will not get set.
   config :skip_empty_columns, :validate => :boolean, :default => false
 
+  # Define whether empty rows could potentially be skipped.
+  # Defaults to false. If set to true, rows containing no value will be tagged with _csvskippedemptyfield.
+  # This tag can referenced by users if they wish to cancel events using an 'if' conditional statement.
+  config :skip_empty_rows, :validate => :boolean, :default => false
+
   # Define a set of datatype conversions to be applied to columns.
   # Possible conversions are integer, float, date, date_time, boolean
   #
@@ -120,7 +129,8 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
 
     if (source = event.get(@source))
       begin
-        values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)
+
+        values = CSV.parse_line(source, :col_sep => @separator, :quote_char => @quote_char)        
 
         if (@autodetect_column_names && @columns.empty?)
           @columns = values
@@ -128,6 +138,17 @@ class LogStash::Filters::CSV < LogStash::Filters::Base
           return
         end
 
+        if (@skip_header && (!@columns.empty?) && (@columns == values))
+          event.cancel
+          return
+        end
+
+        if(@skip_empty_rows && values.nil?)
+          # applies tag to empty rows, users can cancel event referencing this tag in an 'if' conditional statement
+          event.tag("_csvskippedemptyfield")
+          return
+        end
+
         values.each_index do |i|
           unless (@skip_empty_columns && (values[i].nil? || values[i].empty?))
             unless ignore_field?(i)

data/logstash-filter-csv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-csv'
-  s.version         = '3.0.7'
+  s.version         = '3.0.8'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses comma-separated value data into individual fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/csv_spec.rb

@@ -50,6 +50,20 @@ describe LogStash::Filters::CSV do
       end
     end
 
+    describe "empty message" do
+      let(:doc) { "" }
+
+      let(:config) do
+        { "skip_empty_rows" => true }
+      end
+
+      it "skips empty rows" do
+        plugin.filter(event)
+        expect(event.get("tags")).to include("_csvskippedemptyfield") 
+        expect(event).not_to be_cancelled 
+      end
+    end
+
     describe "custom separator" do
       let(:doc) { "big,bird;sesame street" }
 
@@ -147,6 +161,21 @@ describe LogStash::Filters::CSV do
         end
       end
 
+
+      context "parse csv and skip the header" do
+        
+        let(:doc)    { "first_column,second_column,third_column" }
+        let(:config) do
+          { "skip_header" => true,
+            "columns" => ["first_column", "second_column", "third_column"] }
+        end
+
+        it "expects the event to be cancelled" do
+          plugin.filter(event)
+          expect(event).to be_cancelled
+        end
+      end
+
       context "parse csv skipping empty columns" do
 
         let(:doc)    { "val1,,val3" }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-csv
 version: !ruby/object:Gem::Version
-  version: 3.0.7
+  version: 3.0.8
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Parses comma-separated value data into individual fields