logstash-filter-mutate 3.1.3 → 3.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a56a7078074ef2704ec10834c3e7fe7a664ee647
-  data.tar.gz: 726266e6349d2ecbb8570932c19260a029d0b321
+  metadata.gz: 501e5df0525fdfe75cd9037ef0c9359a5508f5b5
+  data.tar.gz: 544b226c746ce230aed79bff95e8570a98b70c72
 SHA512:
-  metadata.gz: c034a0c3708ed37330f6fd18746f6fe47a1f0524417e78483c1f56b6115469b0d37da39621c8b67be9ecc187f6c0a9b30fd3f8f37f181b62ec7573ef5f66af5e
-  data.tar.gz: b9f614d74196e90d942b87f376c8f64d4127d544126331b17b20f105bc30b61e1e9545a3d87ce093cad416b8272ab397d29e2bebb67227aa310e544ba82f9f51
+  metadata.gz: 76bb61eff6123e02ea18749d8e797c1617a018b2bdd66b751c0937372da059c5e23074295531c6d0191ce090d85214b3751d7d4d17ea26e827a855351870a4e4
+  data.tar.gz: 9729bc0839d86afe04cd8d515c9fe624c7e70cefe83d489774d98118ca9f2fe9b3deaf0fc90240f504a67c54c8eca9eabdb892c87b3268b57c843023612b942e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.4
+ - feature: Allow to copy fields.
+
 ## 3.1.3
  - Don't create empty fields when lower/uppercasing a non-existant field
 

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,282 @@
+:plugin: mutate
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Mutate filter plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+The mutate filter allows you to perform general mutations on fields. You
+can rename, remove, replace, and modify fields in your events.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Mutate Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-convert>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-copy>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-gsub>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-join>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-lowercase>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-merge>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-rename>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-replace>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-split>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-strip>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-update>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-uppercase>> |<<array,array>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-convert"]
+===== `convert` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Convert a field's value to a different type, like turning a string to an
+integer. If the field value is an array, all members will be converted.
+If the field is a hash, no action will be taken.
+
+If the conversion type is `boolean`, the acceptable values are:
+
+* **True:** `true`, `t`, `yes`, `y`, and `1`
+* **False:** `false`, `f`, `no`, `n`, and `0`
+
+If a value other than these is provided, it will pass straight through
+and log a warning message.
+
+Valid conversion targets are: integer, float, string, and boolean.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        convert => { "fieldname" => "integer" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-copy"]
+===== `copy` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Copy an existing field to another field. Existing target field will be overriden.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+         copy => { "source_field" => "dest_field" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-gsub"]
+===== `gsub` 
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Convert a string field by applying a regular expression and a replacement.
+If the field is not a string, no action will be taken.
+
+This configuration takes an array consisting of 3 elements per
+field/substitution.
+
+Be aware of escaping any backslash in the config file.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        gsub => [
+          # replace all forward slashes with underscore
+          "fieldname", "/", "_",
+          # replace backslashes, question marks, hashes, and minuses
+          # with a dot "."
+          "fieldname2", "[\\?#-]", "."
+        ]
+      }
+    }
+
+
+[id="plugins-{type}s-{plugin}-join"]
+===== `join` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Join an array with a separator character. Does nothing on non-array fields.
+
+Example:
+[source,ruby]
+   filter {
+     mutate {
+       join => { "fieldname" => "," }
+     }
+   }
+
+[id="plugins-{type}s-{plugin}-lowercase"]
+===== `lowercase` 
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Convert a string to its lowercase equivalent.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        lowercase => [ "fieldname" ]
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-merge"]
+===== `merge` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Merge two fields of arrays or hashes.
+String fields will be automatically be converted into an array, so:
+==========================
+  `array` + `string` will work
+  `string` + `string` will result in an 2 entry array in `dest_field`
+  `array` and `hash` will not work
+==========================
+Example:
+[source,ruby]
+    filter {
+      mutate {
+         merge => { "dest_field" => "added_field" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-rename"]
+===== `rename` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Rename one or more fields.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        # Renames the 'HOSTORIP' field to 'client_ip'
+        rename => { "HOSTORIP" => "client_ip" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-replace"]
+===== `replace` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Replace a field with a new value. The new value can include `%{foo}` strings
+to help you build a new value from other parts of the event.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        replace => { "message" => "%{source_host}: My new message" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-split"]
+===== `split` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Split a field to an array using a separator character. Only works on string
+fields.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+         split => { "fieldname" => "," }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-strip"]
+===== `strip` 
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Strip whitespace from field. NOTE: this only works on leading and trailing whitespace.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+         strip => ["field1", "field2"]
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-update"]
+===== `update` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Update an existing field with a new value. If the field does not exist,
+then no action will be taken.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        update => { "sample" => "My new message" }
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-uppercase"]
+===== `uppercase` 
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Convert a string to its uppercase equivalent.
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        uppercase => [ "fieldname" ]
+      }
+    }
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/mutate.rb

@@ -162,6 +162,17 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   #     }
   config :merge, :validate => :hash
 
+  # Copy an existing field to another field. Existing target field will be overriden.
+  # ==========================
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       mutate {
+  #          copy => { "source_field" => "dest_field" }
+  #       }
+  #     }
+  config :copy, :validate => :hash
+
   TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
   FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
   CONVERT_PREFIX = "convert_".freeze
@@ -212,6 +223,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
     split(event) if @split
     join(event) if @join
     merge(event) if @merge
+    copy(event) if @copy
 
     filter_matched(event)
   end
@@ -427,4 +439,12 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
       end
     end
   end
+
+  def copy(event)
+    @copy.each do |src_field, dest_field|
+      original = event.get(src_field)
+      next if original.nil?
+      event.set(dest_field,LogStash::Util.deep_clone(original))
+    end
+  end
 end

data/logstash-filter-mutate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-mutate'
-  s.version         = '3.1.3'
+  s.version         = '3.1.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The mutate filter allows you to perform general mutations on fields. You can rename, remove, replace, and modify fields in your events."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/filters/integration/multi_stage_spec.rb

@@ -10,7 +10,10 @@ unless LogStash::Environment.const_defined?(:LOGSTASH_HOME)
 end
 
 describe LogStash::Filters::Mutate do
-  let(:pipeline) { LogStash::Pipeline.new(config) }
+  let(:pipeline) do
+    new_pipeline_from_string(config)
+  end
+
   let(:events) do
     arr = event.is_a?(Array) ? event : [event]
     arr.map do |evt|

data/spec/filters/mutate_spec.rb

@@ -178,6 +178,51 @@ describe LogStash::Filters::Mutate do
     end
   end
 
+  describe "#copy" do
+
+    let(:config) do
+      { "copy" => {"field" => "target" } }
+    end
+
+    context "when source field is a string" do
+
+      let(:attrs) { { "field" => "foobar" } }
+
+      it "should deep copy the field" do
+        subject.filter(event)
+        expect(event.get("target")).to eq(event.get("field"))
+        #fields should be independant
+        event.set("field",nil);
+        expect(event.get("target")).not_to eq(event.get("field"))
+      end
+    end
+
+    context "when source field is an array" do
+
+      let(:attrs) { { "field" => ["foo","bar"] } }
+
+      it "should not modify source field nil" do
+        subject.filter(event)
+        expect(event.get("target")).to eq(event.get("field"))
+        #fields should be independant
+        event.set("field",event.get("field") << "baz")
+        expect(event.get("target")).not_to eq(event.get("field"))
+      end
+    end
+
+    context "when source field is a hash" do
+
+      let(:attrs) { { "field" => { "foo" => "bar"} } }
+
+      it "should not modify source field nil" do
+        subject.filter(event)
+        expect(event.get("target")).to eq(event.get("field"))
+        #fields should be independant
+        event.set("[field][foo]","baz")
+        expect(event.get("[target][foo]")).not_to eq(event.get("[field][foo]"))
+      end
+    end
+  end
 end
 
 describe LogStash::Filters::Mutate do
@@ -187,7 +232,7 @@ describe LogStash::Filters::Mutate do
       config <<-CONFIG
         filter {
           mutate {
-            convert => [ "message", "int"] //should be integer
+            convert => [ "message", "int"] #should be integer
           }
         }
       CONFIG

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-mutate
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.1.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-29 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/mutate.rb
 - logstash-filter-mutate.gemspec
 - spec/filters/integration/multi_stage_spec.rb