logstash-filter-translate 3.2.2 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2b0e57299901e68b342a5f9bc038c52a37a214d47302dd3f05718b1c954968
-  data.tar.gz: a946264b317825a0477a1d547faf3843826d7f391eb13d117a5060e9285e8d1f
+  metadata.gz: 77bd945dee5e20dbc7eb6837c655e807373ee8923dcbd498c9fcf5b1065c6661
+  data.tar.gz: 878df6d8dfcc5fa53bb617163b0ddfe6feeb8e876aed9668b337f66dc8472221
 SHA512:
-  metadata.gz: 3f4983b4527a2de1d00682269b374e2828caccdc3f3f1a57bd560d3b098b65a6475c2fb0f6288792805bb0fab2144f95df9dc1c280b228064196eaa2fc30e3b0
-  data.tar.gz: 1c2e0ead540188084abcc23033080e19d009b6651b757cefb6bd1c5ace26801df5d996296f9ddfba9de0d0a0b7a5cb2f7ac672d5cf2b999dc3e45b90f24b809c
+  metadata.gz: d45ec30865d73884f9564f3680fdc1fb230c9fa71507c86d475e6ec0759d13863080f8c237ab34b38ccad08bca0781ab1d8a8523955841db2d0a0334091a9cf3
+  data.tar.gz: 443284f8db270ec6081a2017bd6597733be886643232d97a7630c99e1b194970183e0a7d070ed511bdb1d8069e7f929a991d608798a35d8bd5408878806945d5

data/CHANGELOG.md

@@ -1,3 +1,20 @@
+## 3.3.1
+  - Refactor: reading .csv for JRuby 9.3 compatibility [#94](https://github.com/logstash-plugins/logstash-filter-translate/pull/94)
+
+    NOTE: these changes are essential for the plugin to work properly under Logstash 8.3 and later.
+
+## 3.3.0
+  - Feat: added ECS compatibility mode [#89](https://github.com/logstash-plugins/logstash-filter-translate/pull/89)
+    - deprecated `destination` option in favor of `target` to better align with other plugins
+    - deprecated `field` option in favor of `source` to better align with other plugins
+    - when ECS compatibility is enabled, default behaviour is an in-place translation
+  - Fix: improved error handling - do not rescue potentially fatal (JVM) errors
+
+## 3.2.3
+  - Fix to align with docs - looked-up values are always strings. Coerce better. [#77](https://github.com/logstash-plugins/logstash-filter-translate/pull/77)
+  - Fix bug in dictionary/file the always applied RegexExact, manifested when dictionary keys are not regex compatible [Logstash #9936](https://github.com/elastic/logstash/issues/9936)
+  - Added info to dictionary_path description to explain why integers must be quoted
+
 ## 3.2.2
   - Fix bug in csv_file when LS config has CSV filter plugin specified as well as a csv dictionary.
   [#70](https://github.com/logstash-plugins/logstash-filter-translate/issues/70)

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-translate.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-translate)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-translate.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-translate)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -24,25 +24,27 @@ A general search and replace tool that uses a configured hash
 and/or a file to determine replacement values. Currently supported are
 YAML, JSON, and CSV files. Each dictionary item is a key value pair.
 
-The dictionary entries can be specified in one of two ways: First,
-the `dictionary` configuration item may contain a hash representing
-the mapping. Second, an external file (readable by logstash) may be specified
-in the `dictionary_path` configuration item. These two methods may not be used
-in conjunction; it will produce an error.
+You can specify dictionary entries in one of two ways: 
 
-Operationally, for each event, the value from the `field` setting is tested
+* The `dictionary` configuration item can contain a hash representing
+the mapping. 
+* An external file (readable by logstash) may be specified in the
+`dictionary_path` configuration item. 
+
+These two methods may not be used in conjunction; it will produce an error.
+
+Operationally, for each event, the value from the `source` setting is tested
 against the dictionary and if it matches exactly (or matches a regex when
-`regex` configuration item has been enabled), the matched value is put in
-the `destination` field, but on no match the `fallback` setting string is
-used instead.
+`regex` configuration item has been enabled), the matched value is put in the
+`target` field, but on no match the `fallback` setting string is used instead.
 
 Example:
 [source,ruby]
 ----
     filter {
       translate {
-        field => "[http_status]"
-        destination => "[http_status_description]"
+        source => "[http][response][status_code]"
+        target => "[http_status_description]"
         dictionary => {
           "100" => "Continue"
           "101" => "Switching Protocols"
@@ -77,6 +79,13 @@ Any ongoing modification of the dictionary file should be done using a
 copy/edit/rename or create/rename mechanism to avoid the refresh code from
 processing half-baked dictionary content.
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+The plugin acts as an in-place translator if `source` and `target` are the same
+and does not produce any new event fields.
+This is the default behavior in <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>>.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Translate Filter Configuration Options
 
@@ -88,14 +97,17 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-destination>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-dictionary>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-dictionary_path>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exact>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-fallback>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-field>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-field>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-iterate_on>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-override>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-refresh_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-regex>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-source>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-refresh_behaviour>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -107,12 +119,9 @@ filter plugins.
 ===== `destination`
 
   * Value type is <<string,string>>
-  * Default value is `"translation"`
+  * Deprecated alias for <<plugins-{type}s-{plugin}-target>> setting.
 
-The destination field you wish to populate with the translated code. The default
-is a field named `translation`. Set this to the same value as source if you want
-to do a substitution, in this case filter will allways succeed. This will clobber
-the old value of the source field!
+deprecated[3.3.0, Use <<plugins-{type}s-{plugin}-target>> instead. In 4.0 this setting will be removed.]
 
 [id="plugins-{type}s-{plugin}-dictionary"]
 ===== `dictionary`
@@ -121,7 +130,7 @@ the old value of the source field!
   * Default value is `{}`
 
 The dictionary to use for translation, when specified in the logstash filter
-configuration item (i.e. do not use the `@dictionary_path` file).
+configuration item (i.e. do not use the `dictionary_path` file).
 
 Example:
 [source,ruby]
@@ -146,9 +155,13 @@ NOTE: It is an error to specify both `dictionary` and `dictionary_path`.
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-The full path of the external dictionary file. The format of the table
-should be a standard YAML, JSON, or CSV. Make sure you specify any integer-based keys
-in quotes. For example, the YAML file should look something like this:
+The full path of the external dictionary file. The format of the table should be
+a standard YAML, JSON, or CSV. 
+
+Specify any integer-based keys in quotes. The value taken from the event's
+`source` setting is converted to a string. The lookup dictionary keys must also
+be strings, and the quotes make the integer-based keys function as a string.
+For example, the YAML file should look something like this:
 
 [source,ruby]
 ----
@@ -165,6 +178,20 @@ based on the file extension: `json` for JSON, `yaml` or `yml` for YAML, and
 `csv` for CSV. The CSV format expects exactly two columns, with the first serving
 as the original text (lookup key), and the second column as the translation.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: disabled ECS-compatibility
+    ** `v1`, `v8`: compatibility with the specified major version of the Elastic Common Schema
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+The value of this setting affects the _default_ value of <<plugins-{type}s-{plugin}-target>>.
+
 [id="plugins-{type}s-{plugin}-exact"]
 ===== `exact`
 
@@ -214,14 +241,10 @@ This configuration can be dynamic and include parts of the event using the `%{fi
 [id="plugins-{type}s-{plugin}-field"]
 ===== `field`
 
-  * This is a required setting.
   * Value type is <<string,string>>
-  * There is no default value for this setting.
+  * Deprecated alias for <<plugins-{type}s-{plugin}-source>> setting.
 
-The name of the logstash event field containing the value to be compared for a
-match by the translate filter (e.g. `message`, `host`, `response_code`).
-
-If this field is an array, only the first value will be used.
+deprecated[3.3.0, Use <<plugins-{type}s-{plugin}-source>> instead. In 4.0 this setting will be removed.]
 
 [id="plugins-{type}s-{plugin}-iterate_on"]
 ===== `iterate_on`
@@ -233,14 +256,14 @@ When the value that you need to perform enrichment on is a variable sized array
 then specify the field name in this setting. This setting introduces two modes,
 1) when the value is an array of strings and 2) when the value is an array of
 objects (as in JSON object). +
-In the first mode, you should have the same field name in both `field` and
+In the first mode, you should have the same field name in both `source` and
 `iterate_on`, the result will be an array added to the field specified in the
-`destination` setting. This array will have the looked up value (or the
+`target` setting. This array will have the looked up value (or the
 `fallback` value or nil) in same ordinal position as each sought value. +
 In the second mode, specify the field that has the array of objects in
 `iterate_on` then specify the field in each object that provides the sought value
-with `field` and the field to write the looked up value (or the `fallback` value)
-to with `destination`.
+with `source` and the field to write the looked up value (or the `fallback` value)
+to with `target`.
 
 For a dictionary of:
 [source,csv]
@@ -256,8 +279,8 @@ Example of Mode 1
     filter {
       translate {
         iterate_on => "[collaborator_ids]"
-        field      => "[collaborator_ids]"
-        destination => "[collaborator_names]"
+        source     => "[collaborator_ids]"
+        target     => "[collaborator_names]"
         fallback => "Unknown"
       }
     }
@@ -280,9 +303,9 @@ Example of Mode 2
     filter {
       translate {
         iterate_on => "[collaborators]"
-        field      => "[id]"
-        destination => "[name]"
-        fallback => "Unknown"
+        source     => "[id]"
+        target     => "[name]"
+        fallback   => "Unknown"
       }
     }
 
@@ -332,11 +355,14 @@ After
 ===== `override`
 
   * Value type is <<boolean,boolean>>
-  * Default value is `false`
+  * Default value depends on whether in-place translation is being used
 
-If the destination (or target) field already exists, this configuration item specifies
-whether the filter should skip translation (default) or overwrite the target field
-value with the new translation value.
+If the destination (or target) field already exists, this configuration option controls
+whether the filter skips translation (default behavior) or overwrites the target
+field value with the new translation value.
+
+In case of in-place translation, where `target` is the same as `source` (such as when
+<<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled), overwriting is allowed.
 
 [id="plugins-{type}s-{plugin}-refresh_interval"]
 ===== `refresh_interval`
@@ -354,8 +380,10 @@ A value of zero or less will disable refresh.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-If you'd like to treat dictionary keys as regular expressions, set `regex => true`.
-Note: this is activated only when `exact => true`.
+To treat dictionary keys as regular expressions, set `regex => true`.
+
+Be sure to escape dictionary key strings for use with regex.
+Resources on regex formatting are available online.
 
 [id="plugins-{type}s-{plugin}-refresh_behaviour"]
 ===== `refresh_behaviour`
@@ -369,7 +397,29 @@ same entry will be updated but entries that existed before but not in the new di
 will remain after the merge; `replace` causes the whole dictionary to be replaced
 with a new one (deleting all entries of the old one on update).
 
+[id="plugins-{type}s-{plugin}-source"]
+===== `source`
+
+* This is a required setting.
+* Value type is <<string,string>>
+* There is no default value for this setting.
+
+The name of the logstash event field containing the value to be compared for a
+match by the translate filter (e.g. `message`, `host`, `response_code`).
+
+If this field is an array, only the first value will be used.
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+  * Value type is <<string,string>>
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+    ** ECS Compatibility disabled: `"translation"`
+    ** ECS Compatibility enabled: defaults to the same value as `source`
 
+The target field you wish to populate with the translated code.
+If you set this value to the same value as `source` field, the plugin does a substitution, and
+the filter will succeed. This will clobber the old value of the source field!
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/array_of_maps_value_update.rb

@@ -25,7 +25,7 @@ module LogStash module Filters
         inner = event.get(nested_field)
         next if inner.nil?
         matched = [true, nil]
-        @lookup.fetch_strategy.fetch(inner, matched)
+        @lookup.fetch_strategy.fetch(inner.to_s, matched)
         if matched.first
           event.set(nested_destination, matched.last)
           matches[index] = true

data/lib/logstash/filters/array_of_values_update.rb

@@ -2,30 +2,40 @@
 
 module LogStash module Filters
   class ArrayOfValuesUpdate
+    class CoerceArray
+      def call(source) source; end
+    end
+    class CoerceOther
+      def call(source) Array(source); end
+    end
+
     def initialize(iterate_on, destination, fallback, lookup)
       @iterate_on = iterate_on
       @destination = destination
       @fallback = fallback
       @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
       @lookup = lookup
+      @coercers_table = {}
+      @coercers_table.default = CoerceOther.new
+      @coercers_table[Array] = CoerceArray.new
     end
 
     def test_for_inclusion(event, override)
       # Skip translation in case @destination iterate_on already exists and @override is disabled.
-      return false if event.include?(@destination) && !override
+      return false if !override && event.include?(@destination)
       event.include?(@iterate_on)
     end
 
     def update(event)
       val = event.get(@iterate_on)
-      source = Array(val)
+      source = @coercers_table[val.class].call(val)
       target = Array.new(source.size)
       if @use_fallback
         target.fill(event.sprintf(@fallback))
       end
       source.each_with_index do |inner, index|
         matched = [true, nil]
-        @lookup.fetch_strategy.fetch(inner, matched)
+        @lookup.fetch_strategy.fetch(inner.to_s, matched)
         if matched.first
           target[index] = matched.last
         end

data/lib/logstash/filters/dictionary/csv_file.rb

@@ -6,19 +6,9 @@ module LogStash module Filters module Dictionary
 
     protected
 
-    def initialize_for_file_type
-      @io = StringIO.new("")
-      @csv = ::CSV.new(@io)
-    end
-
     def read_file_into_dictionary
-      # low level CSV read that tries to create as
-      # few intermediate objects as possible
-      # this overwrites the value at key
-      IO.foreach(@dictionary_path, :mode => 'r:bom|utf-8') do |line|
-        @io.string = line
-        k,v = @csv.shift
-        @dictionary[k] = v
+      ::CSV.open(@dictionary_path, 'r:bom|utf-8') do |csv|
+        csv.each { |k,v| @dictionary[k] = v }
       end
     end
   end

data/lib/logstash/filters/dictionary/file.rb

@@ -44,11 +44,12 @@ module LogStash module Filters module Dictionary
       @update_method = method(:merge_dictionary)
       initialize_for_file_type
       args = [@dictionary, rw_lock]
-      if exact
-        @fetch_strategy = regex ? FetchStrategy::File::ExactRegex.new(*args) : FetchStrategy::File::ExactRegex.new(*args)
-      else
-        @fetch_strategy = FetchStrategy::File::RegexUnion.new(*args)
-      end
+      klass = case
+              when exact && regex then FetchStrategy::File::ExactRegex
+              when exact          then FetchStrategy::File::Exact
+              else                     FetchStrategy::File::RegexUnion
+              end
+      @fetch_strategy = klass.new(*args)
       load_dictionary(raise_exception = true)
       stop_scheduler(initial = true)
       start_scheduler unless @refresh_interval <= 0 # disabled, a scheduler interval of zero makes no sense
@@ -131,7 +132,9 @@ module LogStash module Filters module Dictionary
     def loading_exception(e, raise_exception)
       msg = "Translate: #{e.message} when loading dictionary file at #{@dictionary_path}"
       if raise_exception
-        raise DictionaryFileError.new(msg)
+        dfe = DictionaryFileError.new(msg)
+        dfe.set_backtrace(e.backtrace)
+        raise dfe
       else
         @logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
       end

data/lib/logstash/filters/dictionary/memory.rb

@@ -7,11 +7,12 @@ module LogStash module Filters module Dictionary
     attr_reader :dictionary, :fetch_strategy
 
     def initialize(hash, exact, regex)
-      if exact
-        @fetch_strategy = regex ? FetchStrategy::Memory::ExactRegex.new(hash) : FetchStrategy::Memory::Exact.new(hash)
-      else
-        @fetch_strategy = FetchStrategy::Memory::RegexUnion.new(hash)
-      end
+      klass = case
+              when exact && regex then FetchStrategy::Memory::ExactRegex
+              when exact          then FetchStrategy::Memory::Exact
+              else                     FetchStrategy::Memory::RegexUnion
+              end
+      @fetch_strategy = klass.new(hash)
     end
 
     def stop_scheduler