logstash-codec-csv 0.1.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5aaabef583e1e52a16f43545eb47083e02bc951c
-  data.tar.gz: 3aa2c1b325eb41cd65796bd662b02ead367cdaf5
+SHA256:
+  metadata.gz: ea5265ba1dbb2589596a6679118b058d9f73db36c548d8454b2e63a533808908
+  data.tar.gz: 45ddf7937e0f918f72318af5eb99cd3994811ca5f624c5d88071c2da4830790c
 SHA512:
-  metadata.gz: b981ff851e9eb2c7576a58669dd4ffe3c0ce169236063fb103891be5bb3750d2e13f667f07abc20519d82c3fd8c061c9a4e952af8777216ee1213db3c3deadc5
-  data.tar.gz: 8a51e1bff8a64063ae31294c1e498b7b90e54b32bdddf0d5504596d8fe922c6e631c0672aedb82021fe8c11f15f9c93ce09caa349f8e3a3d8ffa1b20a28e6391
+  metadata.gz: fba5feae76e970343fe6541d569cd5701f93d227186c60b13da1e93f41c3f4f24626701257c76169ca94e74a8c56a945a98576f5e169ef7f266e5ee12c8feb4e
+  data.tar.gz: f333ec425b38c6b5d83ea8cbeaaefc032bf2b0ec6de1580b9d08ba0561127f609742fa676408ce46ac366d8cf41d6cc76c9b952ebde55c9d9ee314a921821122

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.0.0
+  - Fixed dependencies to work with logstash v6 and up. Overhauled to match features of the CSV Filter. Improved spec coverage [#4](https://github.com/logstash-plugins/logstash-codec-csv/pull/4)
+## 0.1.5
+  - Fixed asciidoc formatting for example [#3](https://github.com/logstash-plugins/logstash-codec-csv/pull/3)
+
 ## 0.1.4
   - Fix some documentation issues
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elasticsearch.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -20,7 +20,7 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-
+The csv codec takes CSV data, parses it and passes it along.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Csv Codec Configuration Options
@@ -28,6 +28,7 @@ include::{include_path}/plugin_header.asciidoc[]
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-autodetect_column_names>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-autogenerate_column_names>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
 | <<plugins-{type}s-{plugin}-columns>> |<<array,array>>|No
@@ -40,13 +41,22 @@ include::{include_path}/plugin_header.asciidoc[]
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-autodetect_column_names"]
+===== `autodetect_column_names`
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+Define whether column names should be auto-detected from the header column or not.
+Defaults to false.
+
 [id="plugins-{type}s-{plugin}-autogenerate_column_names"]
 ===== `autogenerate_column_names` 
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
 
-Define whether column names should autogenerated or not.
+Define whether column names should be autogenerated or not.
 Defaults to true. If set to false, columns not having a header specified will not be parsed.
 
 [id="plugins-{type}s-{plugin}-charset"]
@@ -65,12 +75,14 @@ The character encoding used in this codec. Examples include "UTF-8" and
   * Value type is <<array,array>>
   * Default value is `[]`
 
+**When decoding:**
 Define a list of column names (in the order they appear in the CSV,
 as if it were a header line). If `columns` is not configured, or there
 are not enough columns specified, the default column names are
-"column1", "column2", etc. In the case that there are more columns
-in the data than specified in this column list, extra columns will be auto-numbered:
-(e.g. "user_defined_1", "user_defined_2", "column3", "column4", etc.)
+"column1", "column2", etc.
+
+**When encoding:**
+List of fields names to include in the encoded CSV, in the order listed.
 
 [id="plugins-{type}s-{plugin}-convert"]
 ===== `convert` 
@@ -79,9 +91,10 @@ in the data than specified in this column list, extra columns will be auto-numbe
   * Default value is `{}`
 
 Define a set of datatype conversions to be applied to columns.
-Possible conversions are integer, float, date, date_time, boolean
+Possible conversions are: `integer`, `float`, `date`, `date_time`, `boolean`
+
 
-# Example:
+*Example*
 [source,ruby]
     filter {
       csv {
@@ -95,10 +108,8 @@ Possible conversions are integer, float, date, date_time, boolean
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-Treats the first line received as the hearder information, this information will
-be used to compose the field names in the generated events. Note this information can
-be reset on demand, useful for example when dealing with new files in the file input
-or new request in the http_poller. Default => false
+When **encoding** in an output plugin, include headers in the encoded CSV
+once per codec lifecyle (not for every event). Default => false
 
 [id="plugins-{type}s-{plugin}-quote_char"]
 ===== `quote_char` 
@@ -127,6 +138,6 @@ Optional.
   * Default value is `false`
 
 Define whether empty columns should be skipped.
-Defaults to false. If set to true, columns containing no value will not get set.
+Defaults to false. If set to true, columns containing no value will not be included.
 
 

data/lib/logstash/codecs/csv.rb

@@ -7,12 +7,13 @@ class LogStash::Codecs::CSV < LogStash::Codecs::Base
 
   config_name "csv"
 
-  # Define a list of column names (in the order they appear in the CSV,
-  # as if it were a header line). If `columns` is not configured, or there
-  # are not enough columns specified, the default column names are
-  # "column1", "column2", etc. In the case that there are more columns
-  # in the data than specified in this column list, extra columns will be auto-numbered:
-  # (e.g. "user_defined_1", "user_defined_2", "column3", "column4", etc.)
+  # When decoding:
+  #   Define a list of column names (in the order they appear in the CSV,
+  #   as if it were a header line). If `columns` is not configured, or there
+  #   are not enough columns specified, the default column names are
+  #   "column1", "column2", etc.
+  # When encoding:
+  #   List of fields names to include in the encoded CSV, in the order listed.
   config :columns, :validate => :array, :default => []
 
   # Define the column separator value. If this is not specified, the default
@@ -25,20 +26,22 @@ class LogStash::Codecs::CSV < LogStash::Codecs::Base
   # Optional.
   config :quote_char, :validate => :string, :default => '"'
 
-  # Treats the first line received as the hearder information, this information will
-  # be used to compose the field names in the generated events. Note this information can
-  # be reset on demand, useful for example when dealing with new files in the file input
-  # or new request in the http_poller. Default => false
+  # When encoding in an output plugin, include headers in the encoded CSV
+  # once per codec lifecyle (not for every event). Default => false
   config :include_headers, :validate => :boolean, :default => false
 
-  # Define whether column names should autogenerated or not.
+  # Define whether column names should be autogenerated or not.
   # Defaults to true. If set to false, columns not having a header specified will not be parsed.
   config :autogenerate_column_names, :validate => :boolean, :default => true
 
   # Define whether empty columns should be skipped.
-  # Defaults to false. If set to true, columns containing no value will not get set.
+  # Defaults to false. If set to true, columns containing no value will not be included.
   config :skip_empty_columns, :validate => :boolean, :default => false
 
+  # Define whether column names should be auto-detected from the header column or not.
+  # Defaults to false.
+  config :autodetect_column_names, :validate => :boolean, :default => false
+
   # Define a set of datatype conversions to be applied to columns.
   # Possible conversions are integer, float, date, date_time, boolean
   #
@@ -51,113 +54,120 @@ class LogStash::Codecs::CSV < LogStash::Codecs::Base
   #     }
   config :convert, :validate => :hash, :default => {}
 
-  ##
-  # List of valid conversion types used for the convert option
-  ##
-  VALID_CONVERT_TYPES = [ "integer", "float", "date", "date_time", "boolean" ].freeze
-
-
   # The character encoding used in this codec. Examples include "UTF-8" and
   # "CP1252".
   config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
 
+  CONVERTERS = {
+    :integer => lambda do |value|
+      CSV::Converters[:integer].call(value)
+    end,
+
+    :float => lambda do |value|
+      CSV::Converters[:float].call(value)
+    end,
+
+    :date => lambda do |value|
+      result = CSV::Converters[:date].call(value)
+      result.is_a?(Date) ? LogStash::Timestamp.new(result.to_time) : result
+    end,
+
+    :date_time => lambda do |value|
+      result = CSV::Converters[:date_time].call(value)
+      result.is_a?(DateTime) ? LogStash::Timestamp.new(result.to_time) : result
+    end,
+
+    :boolean => lambda do |value|
+       value = value.strip.downcase
+       return false if value == "false"
+       return true  if value == "true"
+       return value
+    end
+  }
+  CONVERTERS.default = lambda {|v| v}
+  CONVERTERS.freeze
+
   def register
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
 
     # validate conversion types to be the valid ones.
-    @convert.each_pair do |column, type|
-      if !VALID_CONVERT_TYPES.include?(type)
-        raise LogStash::ConfigurationError, "#{type} is not a valid conversion type."
-      end
+    bad_types = @convert.values.select do |type|
+      !CONVERTERS.has_key?(type.to_sym)
+    end.uniq
+    raise(LogStash::ConfigurationError, "Invalid conversion types: #{bad_types.join(', ')}") unless bad_types.empty?
+
+    # @convert_symbols contains the symbolized types to avoid symbol conversion in the transform method
+    @convert_symbols = @convert.each_with_object({}){|(k, v), result| result[k] = v.to_sym}
+
+    # if the zero byte character is entered in the config, set the value
+    if (@quote_char == "\\x00")
+      @quote_char = "\x00"
     end
 
-    @headers = false
-    @options = { :col_sep => @separator, :quote_char => @quote_char }
+    @logger.debug? && @logger.debug("CSV parsing options", :col_sep => @separator, :quote_char => @quote_char)
   end
 
   def decode(data)
     data = @converter.convert(data)
     begin
-      values = CSV.parse_line(data, @options)
-      if @include_headers && !@headers
-        @headers = true
-        @options[:headers] = values
-      else
-        decoded = {}
-        values.each_with_index do |fields, index|
-          field_name, value = nil, nil
-          if fields.is_a?(String) && !( @skip_empty_columns && fields.nil?)  # No headers
-            next if ignore_field?(index)
-            field_name =  ( !@columns[index].nil? ? @columns[index] : "column#{(index+1)}")
-            value      = fields
-          elsif fields.is_a?(Array) # Got headers
-            field_name = fields[0]
-            value      = fields[1]
+      values = CSV.parse_line(data, :col_sep => @separator, :quote_char => @quote_char)
+
+      if (@autodetect_column_names && @columns.empty?)
+        @columns = values
+        @logger.debug? && @logger.debug("Auto detected the following columns", :columns => @columns.inspect)
+        return
+      end
+
+      decoded = {}
+      values.each_index do |i|
+        unless (@skip_empty_columns && (values[i].nil? || values[i].empty?))
+          unless ignore_field?(i)
+            field_name = @columns[i] || "column#{i + 1}"
+            decoded[field_name] = transform(field_name, values[i])
           end
-          next unless field_name
-          decoded[field_name] = if should_transform?(field_name)
-                                  transform(field_name, value)
-                                else
-                                  value
-                                end
         end
-        yield LogStash::Event.new(decoded) if block_given?
       end
+
+      yield LogStash::Event.new(decoded)
     rescue CSV::MalformedCSVError => e
-      @logger.info("CSV parse failure. Falling back to plain-text", :error => e, :data => data)
-      yield LogStash::Event.new("message" => data, "tags" => ["_csvparsefailure"]) if block_given?
+      @logger.error("CSV parse failure. Falling back to plain-text", :error => e, :data => data)
+      yield LogStash::Event.new("message" => data, "tags" => ["_csvparsefailure"])
     end
   end
 
   def encode(event)
-    csv_data = CSV.generate_line(event.to_hash.values, @options)
-    @on_event.call(event, csv_data)
-  end
+    if @include_headers
+      csv_data = CSV.generate_line(select_keys(event), :col_sep => @separator, :quote_char => @quote_char, :headers => true)
+      @on_event.call(event, csv_data)
+
+      # output headers only once per codec lifecycle
+      @include_headers = false
+    end
 
-  def reset
-    @headers = false
-    @options.delete(:headers)
+    csv_data = CSV.generate_line(select_values(event), :col_sep => @separator, :quote_char => @quote_char)
+    @on_event.call(event, csv_data)
   end
 
   private
 
-  def ignore_field?(index)
-    !@columns[index] && !@autogenerate_column_names
+  def select_values(event)
+    if @columns.empty?
+      event.to_hash.values
+    else
+      @columns.map {|column| event.get(column)}
+    end
   end
 
-  def should_transform?(field_name)
-    !@convert[field_name].nil?
+  def select_keys(event)
+    @columns.empty? ? event.to_hash.keys : @columns
   end
 
-  def transform(field_name, value)
-    transformation = @convert[field_name].to_sym
-    converters[transformation].call(value)
+  def ignore_field?(index)
+    !@columns[index] && !@autogenerate_column_names
   end
 
-  def converters
-    @converters ||= {
-      :integer => lambda do |value|
-        CSV::Converters[:integer].call(value)
-      end,
-      :float => lambda do |value|
-        CSV::Converters[:float].call(value)
-
-      end,
-      :date => lambda do |value|
-        CSV::Converters[:date].call(value)
-
-      end,
-      :date_time => lambda do |value|
-        CSV::Converters[:date_time].call(value)
-      end,
-      :boolean => lambda do |value|
-        value = value.strip.downcase
-        return false if value == "false"
-        return true  if value == "true"
-        return value
-      end
-    }
+  def transform(field_name, value)
+    CONVERTERS[@convert_symbols[field_name]].call(value)
   end
-
-end # class LogStash::Codecs::Plain
+end

data/logstash-codec-csv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-csv'
-  s.version         = '0.1.4'
+  s.version         = '1.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The csv codec take CSV data, parses it and passes it away"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/codecs/csv_spec.rb

@@ -12,14 +12,13 @@ describe LogStash::Codecs::CSV do
   end
 
   describe "decode" do
-
     let(:data) { "big,bird,sesame street" }
 
     it "return an event from CSV data" do
       codec.decode(data) do |event|
-        expect(event["column1"]).to eq("big")
-        expect(event["column2"]).to eq("bird")
-        expect(event["column3"]).to eq("sesame street")
+        expect(event.get("column1")).to eq("big")
+        expect(event.get("column2")).to eq("bird")
+        expect(event.get("column3")).to eq("sesame street")
       end
     end
 
@@ -31,9 +30,9 @@ describe LogStash::Codecs::CSV do
 
       it "extract all the values" do
         codec.decode(data) do |event|
-          expect(event["first"]).to eq("big")
-          expect(event["last"]).to eq("bird")
-          expect(event["address"]).to eq("sesame street")
+          expect(event.get("first")).to eq("big")
+          expect(event.get("last")).to eq("bird")
+          expect(event.get("address")).to eq("sesame street")
         end
       end
 
@@ -48,9 +47,9 @@ describe LogStash::Codecs::CSV do
 
         it "extract all the values" do
           codec.decode(data) do |event|
-            expect(event["custom1"]).to eq("val1")
+            expect(event.get("custom1")).to eq("val1")
             expect(event.to_hash).not_to include("custom2")
-            expect(event["custom3"]).to eq("val3")
+            expect(event.get("custom3")).to eq("val3")
           end
         end
       end
@@ -65,9 +64,9 @@ describe LogStash::Codecs::CSV do
 
         it "extract all the values" do
           codec.decode(data) do |event|
-            expect(event["custom1"]).to eq("val1")
-            expect(event["custom2"]).to eq("val2")
-            expect(event["column3"]).to be_falsey
+            expect(event.get("custom1")).to eq("val1")
+            expect(event.get("custom2")).to eq("val2")
+            expect(event.get("column3")).to be_falsey
           end
         end
       end
@@ -83,8 +82,8 @@ describe LogStash::Codecs::CSV do
 
       it "return an event from CSV data" do
         codec.decode(data) do |event|
-          expect(event["column1"]).to eq("big,bird")
-          expect(event["column2"]).to eq("sesame street")
+          expect(event.get("column1")).to eq("big,bird")
+          expect(event.get("column2")).to eq("sesame street")
         end
       end
     end
@@ -98,9 +97,9 @@ describe LogStash::Codecs::CSV do
 
       it "return an event from CSV data" do
         codec.decode(data) do |event|
-          expect(event["column1"]).to eq("big")
-          expect(event["column2"]).to eq("bird")
-          expect(event["column3"]).to eq("sesame street")
+          expect(event.get("column1")).to eq("big")
+          expect(event.get("column2")).to eq("bird")
+          expect(event.get("column3")).to eq("sesame street")
         end
       end
 
@@ -110,9 +109,9 @@ describe LogStash::Codecs::CSV do
 
         it "return an event from CSV data" do
           codec.decode(data) do |event|
-            expect(event["column1"]).to eq("big")
-            expect(event["column2"]).to eq("bird")
-            expect(event["column3"]).to eq("sesame, street")
+            expect(event.get("column1")).to eq("big")
+            expect(event.get("column2")).to eq("bird")
+            expect(event.get("column3")).to eq("sesame, street")
           end
         end
       end
@@ -125,9 +124,9 @@ describe LogStash::Codecs::CSV do
 
         it "return an event from CSV data" do
           codec.decode(data) do |event|
-            expect(event["column1"]).to eq("big")
-            expect(event["column2"]).to eq("bird")
-            expect(event["column3"]).to eq('"sesame" street')
+            expect(event.get("column1")).to eq("big")
+            expect(event.get("column2")).to eq("bird")
+            expect(event.get("column3")).to eq('"sesame" street')
           end
         end
       end
@@ -144,28 +143,15 @@ describe LogStash::Codecs::CSV do
       end
 
       let(:config) do
-        { "include_headers" => true }
+        { "autodetect_column_names" => true }
       end
 
       it "include header information when requested" do
         codec.decode(data[0]) # Read the headers
         codec.decode(data[1]) do |event|
-          expect(event["size"]).to eq("big")
-          expect(event["animal"]).to eq("bird")
-          expect(event["movie"]).to eq("sesame street")
-        end
-      end
-
-      it "reset headers and fetch the new ones" do
-        data.each do |row|
-          codec.decode(row)
-        end
-        codec.reset
-        codec.decode(new_data[0]) # set the new headers
-        codec.decode(new_data[1]) do |event|
-          expect(event["host"]).to eq("example.com")
-          expect(event["country"]).to eq("germany")
-          expect(event["city"]).to eq("berlin")
+          expect(event.get("size")).to eq("big")
+          expect(event.get("animal")).to eq("bird")
+          expect(event.get("movie")).to eq("sesame street")
         end
       end
     end
@@ -179,9 +165,9 @@ describe LogStash::Codecs::CSV do
 
       it "get converted values to the expected type" do
         codec.decode(data) do |event|        
-          expect(event["column1"]).to eq(1234)
-          expect(event["column2"]).to eq("bird")
-          expect(event["column3"]).to eq(false)
+          expect(event.get("column1")).to eq(1234)
+          expect(event.get("column2")).to eq("bird")
+          expect(event.get("column3")).to eq(false)
         end
       end
 
@@ -194,13 +180,80 @@ describe LogStash::Codecs::CSV do
 
         it "get converted values to the expected type" do
           codec.decode(data) do |event|
-            expect(event["custom1"]).to eq(1234)
-            expect(event["custom2"]).to eq("bird")
-            expect(event["custom3"]).to eq(false)
+            expect(event.get("custom1")).to eq(1234)
+            expect(event.get("custom2")).to eq("bird")
+            expect(event.get("custom3")).to eq(false)
           end
         end
       end
     end
+  end
+
+  describe "encode" do
+    context "not including headers" do
+      let(:event) { LogStash::Event.new({"f1" => "v1", "f2" => "v2"}) }
+
+      context "without columns" do
+        let(:config) do
+          { "include_headers" => false, "columns" => [] }
+        end
 
+        it "should encode to single CSV line" do
+          codec.on_event do |e, d|
+            expect(d.chomp.split(",").sort).to eq("v1,v2,1,#{event.timestamp}".split(",").sort)
+          end
+          codec.encode(event)
+        end
+      end
+
+      context "with columns" do
+        let(:config) do
+          { "include_headers" => false, "columns" => ["f1", "f2"] }
+        end
+
+        it "should encode to single CSV line" do
+          codec.on_event do |e, d|
+            expect(d).to eq("v1,v2\n")
+          end
+          codec.encode(event)
+        end
+      end
+    end
+
+    context "including headers" do
+      let(:event) { LogStash::Event.new({"f1" => "v1", "f2" => "v2"}) }
+
+      context "without columns" do
+        let(:config) do
+          { "include_headers" => true, "columns" => [] }
+        end
+
+        it "should encode to two CSV line" do
+          lines = []
+          codec.on_event do |e, d|
+            lines << d
+          end
+          codec.encode(event)
+          expect(lines[0].chomp.split(",").sort).to eq("f1,f2,@version,@timestamp".split(",").sort)
+          expect(lines[1].chomp.split(",").sort).to eq("v1,v2,1,#{event.timestamp}".split(",").sort)
+        end
+      end
+
+      context "with columns" do
+        let(:config) do
+          { "include_headers" => true, "columns" => ["f1", "f2"] }
+        end
+
+        it "should encode to two CSV line" do
+          lines = []
+          codec.on_event do |e, d|
+            lines << d
+          end
+          codec.encode(event)
+          expect(lines[0]).to eq("f1,f2\n")
+          expect(lines[1]).to eq("v1,v2\n")
+        end
+      end
+    end
   end
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-csv
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 1.0.0
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2020-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.99'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -38,7 +44,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []
 extensions: []
@@ -75,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: The csv codec take CSV data, parses it and passes it away