logstash-filter-translate 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a657937387054b277b294032c5acb3ea34bc8255e3c0e344894ee1160d7d1fa5
-  data.tar.gz: 61c5fdbd1fb8e702109eb494caeffe3100a912e2fb4de98491848969dd7265f1
+  metadata.gz: bac5f7aca89b5e4261183c450ef1fb494ea29a48dc0c32157863a5176238663a
+  data.tar.gz: 24b5b67a25d7cefd48d04fd8d9789583166c748e6901d38a569d61cbc68f069c
 SHA512:
-  metadata.gz: 595cbd7c55aa076f4ce4818654cd02fe0c181527266cc5f808a1bf85c8ad7c6864845f12e7401779aecde04b2017e50282af5b9a137d2284077a70a21055c183
-  data.tar.gz: dc77c15b65f70981a77271c174c2fcc64266d8f1cd59925e31d334c8293c0a683597ee130f40213ced66b3ffd7c3ca00b359b937685f499eb4f27118ad28f512
+  metadata.gz: 38ab5fecaf9a92700cae046e16dc0025b31be2adab0e2dc44609096cef043efff6afe069210010e8db3932c0710aadb29efe3fe879d5ad981345043b838b5ec6
+  data.tar.gz: c0d28aaaf7ab1c08c10ccec79bc25fef1e2c5e86fed295bed81ddcb1f0c84ae184917c867c37e6c9273eff681b5f34bded08b118a3936b279b99f0f1a673c22b

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.5.1
+  - Fixes an issue where failing to load a dictionary could cause the plugin to continue to run with a missing or partially-updated dictionary; this issue was especially noticeable when configured with `refresh_behaviour => replace`, which clears the dictionary before loading the replacement [#112](https://github.com/logstash-plugins/logstash-filter-translate/issues/112).
+
 ## 3.5.0
   - Introduce opt-in "yaml_load_strategy => streaming" to stream parse YAML dictionaries [#106](https://github.com/logstash-plugins/logstash-filter-translate/pull/106)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-translate.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-translate)
+[![Unit Tests](https://github.com/logstash-plugins/logstash-filter-translate/actions/workflows/unit-tests.yml/badge.svg?branch=main)](https://github.com/logstash-plugins/logstash-filter-translate/actions/workflows/unit-tests.yml)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/lib/logstash/filters/dictionary/csv_file.rb

@@ -6,9 +6,11 @@ module LogStash module Filters module Dictionary
 
     protected
 
-    def read_file_into_dictionary
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
       ::CSV.open(@dictionary_path, 'r:bom|utf-8') do |csv|
-        csv.each { |k,v| @dictionary[k] = v }
+        csv.each { |k,v| yield(k,v) }
       end
     end
   end

data/lib/logstash/filters/dictionary/file.rb

@@ -72,27 +72,50 @@ module LogStash module Filters module Dictionary
       # sub class specific initializer
     end
 
-    def read_file_into_dictionary
-      # defined in csv_file, yaml_file and json_file
+    ##
+    # read the dictionary from file into a new hash
+    # @yieldparam [String] key
+    # @yieldparam [Object] value
+    def read_dictionary
+      # defined in concrete implementation
+      fail NotImplementedError, "#{self.class} does not implement `read_dictionary`"
     end
 
     private
 
+    ##
+    # merges the parsed contents of the dictionary file into the current dictionary.
+    # This is an all-or-nothing operation.
+    # @return [void]
     def merge_dictionary
       @write_lock.lock
       begin
-        read_file_into_dictionary
+        changes = {}
+        read_dictionary do |key, value|
+          changes[key] = value unless value == @dictionary[key]
+        end
+        @dictionary.update(changes)
         @fetch_strategy.dictionary_updated
       ensure
         @write_lock.unlock
       end
     end
 
+    ##
+    # replaces the current dictionary with the parsed contents of the dictionary file,
+    # using the existing value objects where possible to limit memory growth.
+    # This is an all-or-nothing operation.
+    # @return [void]
     def replace_dictionary
       @write_lock.lock
       begin
-        @dictionary.clear
-        read_file_into_dictionary
+        replacement_dictionary = {}
+        read_dictionary do |key, value|
+          existing_value = @dictionary[key]
+          # if the value is equivalent to the existing value, use the existing value instead
+          replacement_dictionary[key] = (existing_value == value) ? existing_value : value
+        end
+        @dictionary.replace(replacement_dictionary)
         @fetch_strategy.dictionary_updated
       ensure
         @write_lock.unlock

data/lib/logstash/filters/dictionary/json_file.rb

@@ -4,11 +4,18 @@ require "json"
 module LogStash module Filters module Dictionary
   class JsonFile < File
 
+    EMPTY = Hash.new.freeze
+    private_constant :EMPTY
+
     protected
 
-    def read_file_into_dictionary
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
       content = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
-      @dictionary.update(LogStash::Json.load(content)) unless content.nil? || content.empty?
+      return EMPTY if content.nil? || content.empty?
+
+      LogStash::Json.load(content).each_pair { |key, value| yield(key, value) }
     end
   end
 end end end

data/lib/logstash/filters/dictionary/yaml_file.rb

@@ -13,20 +13,23 @@ module LogStash module Filters module Dictionary
       @yaml_load_strategy = file_type_args[:yaml_load_strategy]
     end
 
-    def read_file_into_dictionary
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
       if @yaml_load_strategy == "one_shot"
         visitor = YamlVisitor.create
         parser = Psych::Parser.new(Psych::TreeBuilder.new)
         parser.code_point_limit = @yaml_code_point_limit
         # low level YAML read that tries to create as
         # few intermediate objects as possible
-        # this overwrites the value at key
         yaml_string = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
         parser.parse(yaml_string, @dictionary_path)
-        visitor.accept_with_dictionary(@dictionary, parser.handler.root)
+        temp_dictionary = {}
+        visitor.accept_with_dictionary(temp_dictionary, parser.handler.root)
+        temp_dictionary.each_pair { |key, value| yield(key, value) }
       else # stream parse it
         parser = StreamingYamlDictParser.new(@dictionary_path, @yaml_code_point_limit)
-        parser.each_pair {|key, value| @dictionary[key] = value }
+        parser.each_pair {|key, value| yield(key, value) }
       end
     end
   end

data/lib/logstash/filters/translate.rb

@@ -249,11 +249,11 @@ class Translate < LogStash::Filters::Base
       @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
     end
 
-    @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
+    logger.debug? && logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
     if @exact
-      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
+      logger.debug? && logger.debug("#{self.class.name}: Dictionary translation method - Exact")
     else
-      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
+      logger.debug? && logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
     end
 
     if @lookup.respond_to?(:reload_dictionary) && @refresh_interval > 0 # a scheduler interval of zero makes no sense

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.5.0'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/filters/scheduling_spec.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+# frozen_string_literal: true
 require 'rspec/wait'
 require "logstash/devutils/rspec/spec_helper"
 require "support/rspec_wait_handler_helper"
@@ -35,6 +36,7 @@ describe LogStash::Filters::Translate do
         file.puts("a,1\nb,2\nc,3\n")
       end
       subject.register
+      allow(subject.lookup).to receive(:logger).and_return(double("LookupLogger").as_null_object)
     end
 
     after do
@@ -71,6 +73,95 @@ describe LogStash::Filters::Translate do
         actions.activate_quietly
         actions.assert_no_errors
       end
+
+      it 'uses the replacement dictionary after reload' do
+        actions = RSpec::Sequencing
+          .run("translate") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('1')
+            expect(event_b.get("translation")).to eq('2')
+            expect(event_c.get("translation")).to eq('3')
+            expect(event_d.get("translation")).to be_nil
+          end
+          .then("modify file with new CSV") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nd,14\n") # changes a+b, removes c, adds d
+            end
+          end
+          .then_after(2, "translate again, ensuring we use the replacement dictionary") do
+
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('11')
+            expect(event_b.get("translation")).to eq('12')
+            expect(event_c.get("translation")).to be_nil # not present in updated dict
+            expect(event_d.get("translation")).to eq('14')
+          end
+          .then("stop") do
+            subject.close
+          end
+
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+
+      context "when replacement file is corrupt" do
+
+        it "logs a warning with the parse error but keeps processing with existing definitions" do
+          actions = RSpec::Sequencing
+            .run("translate") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("modify file with invalid CSV") do
+              dictionary_path.open("w") do |file|
+                file.puts("a,11\nb,12\n\"\xFF".dup.force_encoding("UTF-8")) # intentional broken utf-8
+              end
+            end
+            .then_after(2, "wait for dictionary reload attempt and ensure logs were emitted") do
+              wait_for { subject.lookup.logger }.to have_received(:warn).with(/continuing with old dictionary/, anything)
+            end
+            .then("translate again, ensuring we still use the old dictionary") do
+
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("stop") do
+              subject.close
+            end
+          actions.activate_quietly
+          actions.assert_no_errors
+        end
+      end
     end
 
     context "merge" do
@@ -102,6 +193,92 @@ describe LogStash::Filters::Translate do
         actions.activate_quietly
         actions.assert_no_errors
       end
+
+      it 'uses the merged dictionary after reload' do
+        actions = RSpec::Sequencing
+          .run("translate") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('1')
+            expect(event_b.get("translation")).to eq('2')
+            expect(event_c.get("translation")).to eq('3')
+            expect(event_d.get("translation")).to be_nil
+          end
+          .then("modify file with new CSV") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nd,14\n") # changes a+b, removes c, adds d
+            end
+          end
+          .then_after(2, "translate again, ensuring we use the merged dictionary") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('11')
+            expect(event_b.get("translation")).to eq('12')
+            expect(event_c.get("translation")).to eq('3') # deleted in update, uses old value after merge
+            expect(event_d.get("translation")).to eq('14')
+          end
+          .then("stop") do
+            subject.close
+          end
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+
+      context "when replacement file is corrupt" do
+
+        it "logs a warning with the parse error but keeps processing with existing definitions" do
+          actions = RSpec::Sequencing
+            .run("translate") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("modify file with invalid CSV") do
+              dictionary_path.open("w") do |file|
+                file.puts("a,11\nb,12\n\"\xFF".dup.force_encoding("UTF-8")) # intentional broken utf-8
+              end
+            end
+            .then_after(2, "wait for dictionary reload attempt and ensure logs were emitted") do
+              wait_for { subject.lookup.logger }.to have_received(:warn).with(/continuing with old dictionary/, anything)
+            end
+            .then("translate again, ensuring we still use the old dictionary") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("stop") do
+              subject.close
+            end
+          actions.activate_quietly
+          actions.assert_no_errors
+        end
+      end
     end
   end
 
@@ -127,6 +304,7 @@ describe LogStash::Filters::Translate do
       directory
       wait(1.0).for{Dir.exist?(directory)}.to eq(true)
       LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+      allow(subject).to receive(:logger).and_return(double("Logger").as_null_object)
       subject.register
     end
 
@@ -176,6 +354,7 @@ describe LogStash::Filters::Translate do
       directory
       wait(1.0).for{Dir.exist?(directory)}.to eq(true)
       LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+      allow(subject).to receive(:logger).and_return(double("Logger").as_null_object)
       subject.register
     end
 

data/spec/support/build_huge_dictionaries.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+# frozen_string_literal: true
 
 require "securerandom"
 
@@ -20,10 +21,10 @@ module LogStash module Filters module Dictionary
     tmppath = directory.join("temp_big.json")
     tmppath.open("w") do |file|
       file.puts("{")
-      file.puts('  "foo":"'.concat(SecureRandom.hex(4)).concat('",'))
-      file.puts('  "bar":"'.concat(SecureRandom.hex(4)).concat('",'))
+      file.puts(%Q(  "foo":"#{SecureRandom.hex(4)}",))
+      file.puts(%Q(  "bar":"#{SecureRandom.hex(4)}",))
       size.times do |i|
-        file.puts('  "'.concat(SecureRandom.hex(12)).concat('":"').concat("#{1000000 + i}").concat('",'))
+        file.puts(%Q(  "#{SecureRandom.hex(12)}":"#{1000000 + i}",))
       end
       file.puts('  "baz":"quux"')
       file.puts("}")

data/version

@@ -0,0 +1 @@
+3.5.1

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.5.1
 platform: ruby
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2025-08-04 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -199,6 +199,7 @@ files:
 - spec/fixtures/tag-omap-dict.yml
 - spec/support/build_huge_dictionaries.rb
 - spec/support/rspec_wait_handler_helper.rb
+- version
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -219,7 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.3
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Replaces field contents based on a hash or YAML file
 test_files: