logstash-filter-translate 3.4.3 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 653e221a65b8b528a26f20f1b4c94ce74ecab0ba09e274bba23f0a605819672b
-  data.tar.gz: fae94de89c0a5da2bb87aef35f1aca31fbaefb8717cc292b5f364a886f66e30b
+  metadata.gz: bac5f7aca89b5e4261183c450ef1fb494ea29a48dc0c32157863a5176238663a
+  data.tar.gz: 24b5b67a25d7cefd48d04fd8d9789583166c748e6901d38a569d61cbc68f069c
 SHA512:
-  metadata.gz: 60feb1d75c7579f5b27ee4887f8eec27049f2b54c75b0c554923c958ae46d64261b53a5d494d39c3ac3194d03ceff6bbdff2a92e39e879a0c44d491221b34bd1
-  data.tar.gz: 69c95f3a4313baf6f7bc40557ac73f173405607873dfd483310c8f62da415c9e46d2d24c52133cabe22011bd823f2029272023c6e903c02fc5328a926ef32e72
+  metadata.gz: 38ab5fecaf9a92700cae046e16dc0025b31be2adab0e2dc44609096cef043efff6afe069210010e8db3932c0710aadb29efe3fe879d5ad981345043b838b5ec6
+  data.tar.gz: c0d28aaaf7ab1c08c10ccec79bc25fef1e2c5e86fed295bed81ddcb1f0c84ae184917c867c37e6c9273eff681b5f34bded08b118a3936b279b99f0f1a673c22b

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.5.1
+  - Fixes an issue where failing to load a dictionary could cause the plugin to continue to run with a missing or partially-updated dictionary; this issue was especially noticeable when configured with `refresh_behaviour => replace`, which clears the dictionary before loading the replacement [#112](https://github.com/logstash-plugins/logstash-filter-translate/issues/112).
+
+## 3.5.0
+  - Introduce opt-in "yaml_load_strategy => streaming" to stream parse YAML dictionaries [#106](https://github.com/logstash-plugins/logstash-filter-translate/pull/106)
+
 ## 3.4.3
   - Allow YamlFile's Psych::Parser and Visitor instances to be garbage collected [#104](https://github.com/logstash-plugins/logstash-filter-translate/pull/104)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-translate.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-translate)
+[![Unit Tests](https://github.com/logstash-plugins/logstash-filter-translate/actions/workflows/unit-tests.yml/badge.svg?branch=main)](https://github.com/logstash-plugins/logstash-filter-translate/actions/workflows/unit-tests.yml)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -109,6 +109,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-refresh_behaviour>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-yaml_load_strategy>> |<<string,string>>, one of `["one_shot", "streaming"]`|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -432,5 +433,21 @@ the filter will succeed. This will clobber the old value of the source field!
 The max amount of code points in the YAML file in `dictionary_path`. Please be aware that byte limit depends on the encoding.
 This setting is effective for YAML file only. YAML over the limit throws exception.
 
+[id="plugins-{type}s-{plugin}-yaml_load_strategy"]
+===== `yaml_load_strategy`
+
+* Value can be any of: `one_shot`, `streaming`
+* Default value is `one_shot`
+
+How to load and parse the YAML file. This setting defaults to `one_shot`, which loads the entire
+YAML file into the parser in one go, emitting the final dictionary from the fully parsed YAML document.
+
+Setting to `streaming` will instead instruct the parser to emit one "YAML element" at a time, constructing the dictionary
+during parsing. This mode drastically reduces the amount of memory required to load or refresh the dictionary and it is also faster.
+
+Due to underlying implementation differences this mode only supports basic types such as Arrays, Objects, Strings, numbers and booleans, and does not support tags.
+
+If you have a lot of translate filters with large YAML documents consider changing this setting to `streaming` instead.
+
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/dictionary/csv_file.rb

@@ -6,9 +6,11 @@ module LogStash module Filters module Dictionary
 
     protected
 
-    def read_file_into_dictionary
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
       ::CSV.open(@dictionary_path, 'r:bom|utf-8') do |csv|
-        csv.each { |k,v| @dictionary[k] = v }
+        csv.each { |k,v| yield(k,v) }
       end
     end
   end

data/lib/logstash/filters/dictionary/file.rb

@@ -72,27 +72,50 @@ module LogStash module Filters module Dictionary
       # sub class specific initializer
     end
 
-    def read_file_into_dictionary
-      # defined in csv_file, yaml_file and json_file
+    ##
+    # read the dictionary from file into a new hash
+    # @yieldparam [String] key
+    # @yieldparam [Object] value
+    def read_dictionary
+      # defined in concrete implementation
+      fail NotImplementedError, "#{self.class} does not implement `read_dictionary`"
     end
 
     private
 
+    ##
+    # merges the parsed contents of the dictionary file into the current dictionary.
+    # This is an all-or-nothing operation.
+    # @return [void]
     def merge_dictionary
       @write_lock.lock
       begin
-        read_file_into_dictionary
+        changes = {}
+        read_dictionary do |key, value|
+          changes[key] = value unless value == @dictionary[key]
+        end
+        @dictionary.update(changes)
         @fetch_strategy.dictionary_updated
       ensure
         @write_lock.unlock
       end
     end
 
+    ##
+    # replaces the current dictionary with the parsed contents of the dictionary file,
+    # using the existing value objects where possible to limit memory growth.
+    # This is an all-or-nothing operation.
+    # @return [void]
     def replace_dictionary
       @write_lock.lock
       begin
-        @dictionary.clear
-        read_file_into_dictionary
+        replacement_dictionary = {}
+        read_dictionary do |key, value|
+          existing_value = @dictionary[key]
+          # if the value is equivalent to the existing value, use the existing value instead
+          replacement_dictionary[key] = (existing_value == value) ? existing_value : value
+        end
+        @dictionary.replace(replacement_dictionary)
         @fetch_strategy.dictionary_updated
       ensure
         @write_lock.unlock

data/lib/logstash/filters/dictionary/json_file.rb

@@ -4,11 +4,18 @@ require "json"
 module LogStash module Filters module Dictionary
   class JsonFile < File
 
+    EMPTY = Hash.new.freeze
+    private_constant :EMPTY
+
     protected
 
-    def read_file_into_dictionary
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
       content = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
-      @dictionary.update(LogStash::Json.load(content)) unless content.nil? || content.empty?
+      return EMPTY if content.nil? || content.empty?
+
+      LogStash::Json.load(content).each_pair { |key, value| yield(key, value) }
     end
   end
 end end end

data/lib/logstash/filters/dictionary/streaming_yaml_parser.rb

@@ -0,0 +1,111 @@
+module LogStash module Filters module Dictionary
+  class StreamingYamlDictParser
+    def snakeYamlEngineV2
+      Java::org.snakeyaml.engine.v2
+    end
+
+    def snakeYamlEngineV2Events
+      snakeYamlEngineV2.events
+    end
+
+    def initialize(filename, yaml_code_point_limit)
+      settings = snakeYamlEngineV2.api.LoadSettings.builder
+        .set_code_point_limit(yaml_code_point_limit)
+        .build
+
+      stream = Java::java.io.FileInputStream.new(filename)
+      reader = Java::java.io.InputStreamReader.new(stream, Java::java.nio.charset.StandardCharsets::UTF_8)
+      stream_reader = snakeYamlEngineV2.scanner.StreamReader.new(reader, settings)
+
+      @parser = snakeYamlEngineV2.parser.ParserImpl.new(stream_reader, settings)
+
+      skip_until(snakeYamlEngineV2Events.MappingStartEvent)
+    end
+
+
+    def each_pair
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.MappingEndEvent)
+        key = parse_node
+        value = parse_node
+        yield(key, value)
+      end
+    end
+
+    private
+
+    def next_event
+      @parser.next
+    ensure
+      nil
+    end
+
+    def peek_event
+      @parser.peek_event
+    end
+
+    def skip_until(event_class)
+      while @parser.has_next
+        evt = @parser.next
+        return if event_class === evt
+      end
+    end
+
+    def parse_node
+      event = next_event
+
+      case event
+      when snakeYamlEngineV2Events.ScalarEvent
+        parse_scalar(event)
+      when snakeYamlEngineV2Events.MappingStartEvent
+        parse_mapping
+      when snakeYamlEngineV2Events.SequenceStartEvent
+        parse_sequence
+      else
+        raise "Unexpected event: #{event.class}"
+      end
+    end
+
+    def parse_mapping
+      hash = {}
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.MappingEndEvent)
+        key = parse_node
+        value = parse_node
+        hash[key] = value
+      end
+      next_event
+      hash
+    end
+
+    def parse_sequence
+      array = []
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.SequenceEndEvent)
+        array << parse_node
+      end
+      next_event
+      array
+    end
+    
+    def parse_scalar(scalar)
+      value = scalar.value
+      # return quoted scalars as they are
+      # e.g. don't convert "true" to true
+      return value unless scalar.is_plain
+
+      # otherwise let's do some checking and conversions
+      case value
+      when 'null', '', '~' then nil
+      when 'true' then true
+      when 'false' then false
+      else
+        # Try to convert to integer or float
+        if value.match?(/\A-?\d+\z/)
+          value.to_i
+        elsif value.match?(/\A-?\d+\.\d+\z/)
+          value.to_f
+        else
+          value
+        end
+      end
+    end
+  end
+end end end

data/lib/logstash/filters/dictionary/yaml_file.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require_relative "yaml_visitor"
+require_relative "streaming_yaml_parser"
 
 module LogStash module Filters module Dictionary
   class YamlFile < File
@@ -9,18 +10,27 @@ module LogStash module Filters module Dictionary
 
     def initialize_for_file_type(**file_type_args)
       @yaml_code_point_limit = file_type_args[:yaml_code_point_limit]
+      @yaml_load_strategy = file_type_args[:yaml_load_strategy]
     end
 
-    def read_file_into_dictionary
-      visitor = YamlVisitor.create
-      parser = Psych::Parser.new(Psych::TreeBuilder.new)
-      parser.code_point_limit = @yaml_code_point_limit
-      # low level YAML read that tries to create as
-      # few intermediate objects as possible
-      # this overwrites the value at key
-      yaml_string = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
-      parser.parse(yaml_string, @dictionary_path)
-      visitor.accept_with_dictionary(@dictionary, parser.handler.root)
+    def read_dictionary
+      return enum_for(:read_dictionary) unless block_given?
+
+      if @yaml_load_strategy == "one_shot"
+        visitor = YamlVisitor.create
+        parser = Psych::Parser.new(Psych::TreeBuilder.new)
+        parser.code_point_limit = @yaml_code_point_limit
+        # low level YAML read that tries to create as
+        # few intermediate objects as possible
+        yaml_string = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
+        parser.parse(yaml_string, @dictionary_path)
+        temp_dictionary = {}
+        visitor.accept_with_dictionary(temp_dictionary, parser.handler.root)
+        temp_dictionary.each_pair { |key, value| yield(key, value) }
+      else # stream parse it
+        parser = StreamingYamlDictParser.new(@dictionary_path, @yaml_code_point_limit)
+        parser.each_pair {|key, value| yield(key, value) }
+      end
     end
   end
 end end end

data/lib/logstash/filters/translate.rb

@@ -108,6 +108,10 @@ class Translate < LogStash::Filters::Base
   # The default value is 128MB for code points of size 1 byte
   config :yaml_dictionary_code_point_limit, :validate => :number
 
+  # either load the entire yaml into memory before generating the in-memory dictionary
+  # alternatively "streaming" will gradually build the dictionary with little memory overhead
+  config :yaml_load_strategy, :validate => ["streaming", "one_shot"], :default => "one_shot"
+
   # When using a dictionary file, this setting will indicate how frequently
   # (in seconds) logstash will check the dictionary file for updates.
   config :refresh_interval, :validate => :number, :default => 300
@@ -195,7 +199,7 @@ class Translate < LogStash::Filters::Base
         if @yaml_dictionary_code_point_limit <= 0
           raise LogStash::ConfigurationError, "Please set a positive number in `yaml_dictionary_code_point_limit => #{@yaml_dictionary_code_point_limit}`."
         else
-          @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex, yaml_code_point_limit: @yaml_dictionary_code_point_limit)
+          @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex, yaml_code_point_limit: @yaml_dictionary_code_point_limit, yaml_load_strategy: @yaml_load_strategy)
         end
       elsif @yaml_dictionary_code_point_limit != nil
         raise LogStash::ConfigurationError, "Please remove `yaml_dictionary_code_point_limit` for dictionary file in JSON or CSV format"
@@ -245,11 +249,11 @@ class Translate < LogStash::Filters::Base
       @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
     end
 
-    @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
+    logger.debug? && logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
     if @exact
-      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
+      logger.debug? && logger.debug("#{self.class.name}: Dictionary translation method - Exact")
     else
-      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
+      logger.debug? && logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
     end
 
     if @lookup.respond_to?(:reload_dictionary) && @refresh_interval > 0 # a scheduler interval of zero makes no sense

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.4.3'
+  s.version         = ::File.read('version').split("\n").first
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "version", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/filters/scheduling_spec.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+# frozen_string_literal: true
 require 'rspec/wait'
 require "logstash/devutils/rspec/spec_helper"
 require "support/rspec_wait_handler_helper"
@@ -35,6 +36,7 @@ describe LogStash::Filters::Translate do
         file.puts("a,1\nb,2\nc,3\n")
       end
       subject.register
+      allow(subject.lookup).to receive(:logger).and_return(double("LookupLogger").as_null_object)
     end
 
     after do
@@ -71,6 +73,95 @@ describe LogStash::Filters::Translate do
         actions.activate_quietly
         actions.assert_no_errors
       end
+
+      it 'uses the replacement dictionary after reload' do
+        actions = RSpec::Sequencing
+          .run("translate") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('1')
+            expect(event_b.get("translation")).to eq('2')
+            expect(event_c.get("translation")).to eq('3')
+            expect(event_d.get("translation")).to be_nil
+          end
+          .then("modify file with new CSV") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nd,14\n") # changes a+b, removes c, adds d
+            end
+          end
+          .then_after(2, "translate again, ensuring we use the replacement dictionary") do
+
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('11')
+            expect(event_b.get("translation")).to eq('12')
+            expect(event_c.get("translation")).to be_nil # not present in updated dict
+            expect(event_d.get("translation")).to eq('14')
+          end
+          .then("stop") do
+            subject.close
+          end
+
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+
+      context "when replacement file is corrupt" do
+
+        it "logs a warning with the parse error but keeps processing with existing definitions" do
+          actions = RSpec::Sequencing
+            .run("translate") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("modify file with invalid CSV") do
+              dictionary_path.open("w") do |file|
+                file.puts("a,11\nb,12\n\"\xFF".dup.force_encoding("UTF-8")) # intentional broken utf-8
+              end
+            end
+            .then_after(2, "wait for dictionary reload attempt and ensure logs were emitted") do
+              wait_for { subject.lookup.logger }.to have_received(:warn).with(/continuing with old dictionary/, anything)
+            end
+            .then("translate again, ensuring we still use the old dictionary") do
+
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("stop") do
+              subject.close
+            end
+          actions.activate_quietly
+          actions.assert_no_errors
+        end
+      end
     end
 
     context "merge" do
@@ -102,6 +193,92 @@ describe LogStash::Filters::Translate do
         actions.activate_quietly
         actions.assert_no_errors
       end
+
+      it 'uses the merged dictionary after reload' do
+        actions = RSpec::Sequencing
+          .run("translate") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('1')
+            expect(event_b.get("translation")).to eq('2')
+            expect(event_c.get("translation")).to eq('3')
+            expect(event_d.get("translation")).to be_nil
+          end
+          .then("modify file with new CSV") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nd,14\n") # changes a+b, removes c, adds d
+            end
+          end
+          .then_after(2, "translate again, ensuring we use the merged dictionary") do
+            event_a = LogStash::Event.new("status" => "a" )
+            event_b = LogStash::Event.new("status" => "b" )
+            event_c = LogStash::Event.new("status" => "c" )
+            event_d = LogStash::Event.new("status" => "d" )
+
+            subject.multi_filter([event_a, event_b, event_c, event_d])
+
+            expect(event_a.get("translation")).to eq('11')
+            expect(event_b.get("translation")).to eq('12')
+            expect(event_c.get("translation")).to eq('3') # deleted in update, uses old value after merge
+            expect(event_d.get("translation")).to eq('14')
+          end
+          .then("stop") do
+            subject.close
+          end
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+
+      context "when replacement file is corrupt" do
+
+        it "logs a warning with the parse error but keeps processing with existing definitions" do
+          actions = RSpec::Sequencing
+            .run("translate") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("modify file with invalid CSV") do
+              dictionary_path.open("w") do |file|
+                file.puts("a,11\nb,12\n\"\xFF".dup.force_encoding("UTF-8")) # intentional broken utf-8
+              end
+            end
+            .then_after(2, "wait for dictionary reload attempt and ensure logs were emitted") do
+              wait_for { subject.lookup.logger }.to have_received(:warn).with(/continuing with old dictionary/, anything)
+            end
+            .then("translate again, ensuring we still use the old dictionary") do
+              event_a = LogStash::Event.new("status" => "a" )
+              event_b = LogStash::Event.new("status" => "b" )
+              event_c = LogStash::Event.new("status" => "c" )
+              event_d = LogStash::Event.new("status" => "d" )
+
+              subject.multi_filter([event_a, event_b, event_c, event_d])
+
+              expect(event_a.get("translation")).to eq('1')
+              expect(event_b.get("translation")).to eq('2')
+              expect(event_c.get("translation")).to eq('3')
+              expect(event_d.get("translation")).to be_nil
+            end
+            .then("stop") do
+              subject.close
+            end
+          actions.activate_quietly
+          actions.assert_no_errors
+        end
+      end
     end
   end
 
@@ -127,6 +304,7 @@ describe LogStash::Filters::Translate do
       directory
       wait(1.0).for{Dir.exist?(directory)}.to eq(true)
       LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+      allow(subject).to receive(:logger).and_return(double("Logger").as_null_object)
       subject.register
     end
 
@@ -176,6 +354,7 @@ describe LogStash::Filters::Translate do
       directory
       wait(1.0).for{Dir.exist?(directory)}.to eq(true)
       LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+      allow(subject).to receive(:logger).and_return(double("Logger").as_null_object)
       subject.register
     end
 

data/spec/filters/translate_spec.rb

@@ -238,6 +238,21 @@ describe LogStash::Filters::Translate do
         subject.filter(event)
         expect(event.get("translation")).to eq(1)
       end
+
+      describe "yaml_load_strategy" do
+        let(:one_shot_parse_filter) { subject }
+        let(:streaming_parse_filter) { described_class.new(config.merge("yaml_load_strategy" => 'streaming')) }
+
+        before(:each) do
+          subject.register
+          streaming_parse_filter.register
+        end
+        let(:one_shot_dictionary) { one_shot_parse_filter.lookup.dictionary }
+        let(:streaming_dictionary) { streaming_parse_filter.lookup.dictionary }
+        it "produces an equivalent dictionary for both strategies" do
+          expect(one_shot_dictionary).to eq(streaming_dictionary)
+        end
+      end
     end
 
     describe "when using a yml dictionary with code point limit" do
@@ -246,14 +261,16 @@ describe LogStash::Filters::Translate do
           "source"      => "status",
           "target"      => "translation",
           "dictionary_path"  => dictionary_path,
-          "yaml_dictionary_code_point_limit" => dictionary_size  # the file is 18 bytes
+          "yaml_dictionary_code_point_limit" => codepoint_limit
         }
       end
       let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
+      let(:dictionary_size) { IO.read(dictionary_path).size }
       let(:event) { LogStash::Event.new("status" => "a") }
+      let(:codepoint_limit) { dictionary_size }
 
-      context "dictionary is over limit" do
-        let(:dictionary_size) { 17 }
+      context "codepoint limit under dictionary size" do
+        let(:codepoint_limit) { dictionary_size / 2 }
 
         it "raises exception" do
           expect { subject.register }.to raise_error(/The incoming YAML document exceeds/)
@@ -261,8 +278,6 @@ describe LogStash::Filters::Translate do
       end
 
       context "dictionary is within limit" do
-        let(:dictionary_size) { 18 }
-
         it "returns the exact translation" do
           subject.register
           subject.filter(event)

data/spec/fixtures/dict.yml

@@ -1,3 +1,4 @@
 a : 1
 b : 2
 c : 3
+d : { "e": [1, "hello", true, "false", "1", "1.1"] }

data/spec/support/build_huge_dictionaries.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+# frozen_string_literal: true
 
 require "securerandom"
 
@@ -20,10 +21,10 @@ module LogStash module Filters module Dictionary
     tmppath = directory.join("temp_big.json")
     tmppath.open("w") do |file|
       file.puts("{")
-      file.puts('  "foo":"'.concat(SecureRandom.hex(4)).concat('",'))
-      file.puts('  "bar":"'.concat(SecureRandom.hex(4)).concat('",'))
+      file.puts(%Q(  "foo":"#{SecureRandom.hex(4)}",))
+      file.puts(%Q(  "bar":"#{SecureRandom.hex(4)}",))
       size.times do |i|
-        file.puts('  "'.concat(SecureRandom.hex(12)).concat('":"').concat("#{1000000 + i}").concat('",'))
+        file.puts(%Q(  "#{SecureRandom.hex(12)}":"#{1000000 + i}",))
       end
       file.puts('  "baz":"quux"')
       file.puts("}")

data/version

@@ -0,0 +1 @@
+3.5.1

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.4.3
+  version: 3.5.1
 platform: ruby
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-07-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,12 +30,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -45,12 +44,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-validator_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-validator_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,12 +58,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-deprecation_logger_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-deprecation_logger_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -73,12 +72,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-scheduler
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -87,12 +86,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: psych
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.0
-  name: psych
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,12 +100,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.1.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -115,12 +114,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-sequencing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-sequencing
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -129,12 +128,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-wait
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-wait
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -143,12 +142,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: benchmark-ips
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -177,6 +176,7 @@ files:
 - lib/logstash/filters/dictionary/file.rb
 - lib/logstash/filters/dictionary/json_file.rb
 - lib/logstash/filters/dictionary/memory.rb
+- lib/logstash/filters/dictionary/streaming_yaml_parser.rb
 - lib/logstash/filters/dictionary/yaml_file.rb
 - lib/logstash/filters/dictionary/yaml_visitor.rb
 - lib/logstash/filters/fetch_strategy/file.rb
@@ -199,13 +199,13 @@ files:
 - spec/fixtures/tag-omap-dict.yml
 - spec/support/build_huge_dictionaries.rb
 - spec/support/rspec_wait_handler_helper.rb
+- version
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -220,8 +220,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Replaces field contents based on a hash or YAML file
 test_files: