logstash-filter-translate 3.4.3 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 653e221a65b8b528a26f20f1b4c94ce74ecab0ba09e274bba23f0a605819672b
-  data.tar.gz: fae94de89c0a5da2bb87aef35f1aca31fbaefb8717cc292b5f364a886f66e30b
+  metadata.gz: a657937387054b277b294032c5acb3ea34bc8255e3c0e344894ee1160d7d1fa5
+  data.tar.gz: 61c5fdbd1fb8e702109eb494caeffe3100a912e2fb4de98491848969dd7265f1
 SHA512:
-  metadata.gz: 60feb1d75c7579f5b27ee4887f8eec27049f2b54c75b0c554923c958ae46d64261b53a5d494d39c3ac3194d03ceff6bbdff2a92e39e879a0c44d491221b34bd1
-  data.tar.gz: 69c95f3a4313baf6f7bc40557ac73f173405607873dfd483310c8f62da415c9e46d2d24c52133cabe22011bd823f2029272023c6e903c02fc5328a926ef32e72
+  metadata.gz: 595cbd7c55aa076f4ce4818654cd02fe0c181527266cc5f808a1bf85c8ad7c6864845f12e7401779aecde04b2017e50282af5b9a137d2284077a70a21055c183
+  data.tar.gz: dc77c15b65f70981a77271c174c2fcc64266d8f1cd59925e31d334c8293c0a683597ee130f40213ced66b3ffd7c3ca00b359b937685f499eb4f27118ad28f512

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.5.0
+  - Introduce opt-in "yaml_load_strategy => streaming" to stream parse YAML dictionaries [#106](https://github.com/logstash-plugins/logstash-filter-translate/pull/106)
+
 ## 3.4.3
   - Allow YamlFile's Psych::Parser and Visitor instances to be garbage collected [#104](https://github.com/logstash-plugins/logstash-filter-translate/pull/104)
 

data/docs/index.asciidoc

@@ -109,6 +109,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-refresh_behaviour>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-yaml_dictionary_code_point_limit>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-yaml_load_strategy>> |<<string,string>>, one of `["one_shot", "streaming"]`|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -432,5 +433,21 @@ the filter will succeed. This will clobber the old value of the source field!
 The max amount of code points in the YAML file in `dictionary_path`. Please be aware that byte limit depends on the encoding.
 This setting is effective for YAML file only. YAML over the limit throws exception.
 
+[id="plugins-{type}s-{plugin}-yaml_load_strategy"]
+===== `yaml_load_strategy`
+
+* Value can be any of: `one_shot`, `streaming`
+* Default value is `one_shot`
+
+How to load and parse the YAML file. This setting defaults to `one_shot`, which loads the entire
+YAML file into the parser in one go, emitting the final dictionary from the fully parsed YAML document.
+
+Setting to `streaming` will instead instruct the parser to emit one "YAML element" at a time, constructing the dictionary
+during parsing. This mode drastically reduces the amount of memory required to load or refresh the dictionary and it is also faster.
+
+Due to underlying implementation differences this mode only supports basic types such as Arrays, Objects, Strings, numbers and booleans, and does not support tags.
+
+If you have a lot of translate filters with large YAML documents consider changing this setting to `streaming` instead.
+
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/dictionary/streaming_yaml_parser.rb

@@ -0,0 +1,111 @@
+module LogStash module Filters module Dictionary
+  class StreamingYamlDictParser
+    def snakeYamlEngineV2
+      Java::org.snakeyaml.engine.v2
+    end
+
+    def snakeYamlEngineV2Events
+      snakeYamlEngineV2.events
+    end
+
+    def initialize(filename, yaml_code_point_limit)
+      settings = snakeYamlEngineV2.api.LoadSettings.builder
+        .set_code_point_limit(yaml_code_point_limit)
+        .build
+
+      stream = Java::java.io.FileInputStream.new(filename)
+      reader = Java::java.io.InputStreamReader.new(stream, Java::java.nio.charset.StandardCharsets::UTF_8)
+      stream_reader = snakeYamlEngineV2.scanner.StreamReader.new(reader, settings)
+
+      @parser = snakeYamlEngineV2.parser.ParserImpl.new(stream_reader, settings)
+
+      skip_until(snakeYamlEngineV2Events.MappingStartEvent)
+    end
+
+
+    def each_pair
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.MappingEndEvent)
+        key = parse_node
+        value = parse_node
+        yield(key, value)
+      end
+    end
+
+    private
+
+    def next_event
+      @parser.next
+    ensure
+      nil
+    end
+
+    def peek_event
+      @parser.peek_event
+    end
+
+    def skip_until(event_class)
+      while @parser.has_next
+        evt = @parser.next
+        return if event_class === evt
+      end
+    end
+
+    def parse_node
+      event = next_event
+
+      case event
+      when snakeYamlEngineV2Events.ScalarEvent
+        parse_scalar(event)
+      when snakeYamlEngineV2Events.MappingStartEvent
+        parse_mapping
+      when snakeYamlEngineV2Events.SequenceStartEvent
+        parse_sequence
+      else
+        raise "Unexpected event: #{event.class}"
+      end
+    end
+
+    def parse_mapping
+      hash = {}
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.MappingEndEvent)
+        key = parse_node
+        value = parse_node
+        hash[key] = value
+      end
+      next_event
+      hash
+    end
+
+    def parse_sequence
+      array = []
+      while peek_event && !peek_event.is_a?(snakeYamlEngineV2Events.SequenceEndEvent)
+        array << parse_node
+      end
+      next_event
+      array
+    end
+    
+    def parse_scalar(scalar)
+      value = scalar.value
+      # return quoted scalars as they are
+      # e.g. don't convert "true" to true
+      return value unless scalar.is_plain
+
+      # otherwise let's do some checking and conversions
+      case value
+      when 'null', '', '~' then nil
+      when 'true' then true
+      when 'false' then false
+      else
+        # Try to convert to integer or float
+        if value.match?(/\A-?\d+\z/)
+          value.to_i
+        elsif value.match?(/\A-?\d+\.\d+\z/)
+          value.to_f
+        else
+          value
+        end
+      end
+    end
+  end
+end end end

data/lib/logstash/filters/dictionary/yaml_file.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require_relative "yaml_visitor"
+require_relative "streaming_yaml_parser"
 
 module LogStash module Filters module Dictionary
   class YamlFile < File
@@ -9,18 +10,24 @@ module LogStash module Filters module Dictionary
 
     def initialize_for_file_type(**file_type_args)
       @yaml_code_point_limit = file_type_args[:yaml_code_point_limit]
+      @yaml_load_strategy = file_type_args[:yaml_load_strategy]
     end
 
     def read_file_into_dictionary
-      visitor = YamlVisitor.create
-      parser = Psych::Parser.new(Psych::TreeBuilder.new)
-      parser.code_point_limit = @yaml_code_point_limit
-      # low level YAML read that tries to create as
-      # few intermediate objects as possible
-      # this overwrites the value at key
-      yaml_string = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
-      parser.parse(yaml_string, @dictionary_path)
-      visitor.accept_with_dictionary(@dictionary, parser.handler.root)
+      if @yaml_load_strategy == "one_shot"
+        visitor = YamlVisitor.create
+        parser = Psych::Parser.new(Psych::TreeBuilder.new)
+        parser.code_point_limit = @yaml_code_point_limit
+        # low level YAML read that tries to create as
+        # few intermediate objects as possible
+        # this overwrites the value at key
+        yaml_string = IO.read(@dictionary_path, :mode => 'r:bom|utf-8')
+        parser.parse(yaml_string, @dictionary_path)
+        visitor.accept_with_dictionary(@dictionary, parser.handler.root)
+      else # stream parse it
+        parser = StreamingYamlDictParser.new(@dictionary_path, @yaml_code_point_limit)
+        parser.each_pair {|key, value| @dictionary[key] = value }
+      end
     end
   end
 end end end

data/lib/logstash/filters/translate.rb

@@ -108,6 +108,10 @@ class Translate < LogStash::Filters::Base
   # The default value is 128MB for code points of size 1 byte
   config :yaml_dictionary_code_point_limit, :validate => :number
 
+  # either load the entire yaml into memory before generating the in-memory dictionary
+  # alternatively "streaming" will gradually build the dictionary with little memory overhead
+  config :yaml_load_strategy, :validate => ["streaming", "one_shot"], :default => "one_shot"
+
   # When using a dictionary file, this setting will indicate how frequently
   # (in seconds) logstash will check the dictionary file for updates.
   config :refresh_interval, :validate => :number, :default => 300
@@ -195,7 +199,7 @@ class Translate < LogStash::Filters::Base
         if @yaml_dictionary_code_point_limit <= 0
           raise LogStash::ConfigurationError, "Please set a positive number in `yaml_dictionary_code_point_limit => #{@yaml_dictionary_code_point_limit}`."
         else
-          @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex, yaml_code_point_limit: @yaml_dictionary_code_point_limit)
+          @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex, yaml_code_point_limit: @yaml_dictionary_code_point_limit, yaml_load_strategy: @yaml_load_strategy)
         end
       elsif @yaml_dictionary_code_point_limit != nil
         raise LogStash::ConfigurationError, "Please remove `yaml_dictionary_code_point_limit` for dictionary file in JSON or CSV format"

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.4.3'
+  s.version         = '3.5.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/translate_spec.rb

@@ -238,6 +238,21 @@ describe LogStash::Filters::Translate do
         subject.filter(event)
         expect(event.get("translation")).to eq(1)
       end
+
+      describe "yaml_load_strategy" do
+        let(:one_shot_parse_filter) { subject }
+        let(:streaming_parse_filter) { described_class.new(config.merge("yaml_load_strategy" => 'streaming')) }
+
+        before(:each) do
+          subject.register
+          streaming_parse_filter.register
+        end
+        let(:one_shot_dictionary) { one_shot_parse_filter.lookup.dictionary }
+        let(:streaming_dictionary) { streaming_parse_filter.lookup.dictionary }
+        it "produces an equivalent dictionary for both strategies" do
+          expect(one_shot_dictionary).to eq(streaming_dictionary)
+        end
+      end
     end
 
     describe "when using a yml dictionary with code point limit" do
@@ -246,14 +261,16 @@ describe LogStash::Filters::Translate do
           "source"      => "status",
           "target"      => "translation",
           "dictionary_path"  => dictionary_path,
-          "yaml_dictionary_code_point_limit" => dictionary_size  # the file is 18 bytes
+          "yaml_dictionary_code_point_limit" => codepoint_limit
         }
       end
       let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
+      let(:dictionary_size) { IO.read(dictionary_path).size }
       let(:event) { LogStash::Event.new("status" => "a") }
+      let(:codepoint_limit) { dictionary_size }
 
-      context "dictionary is over limit" do
-        let(:dictionary_size) { 17 }
+      context "codepoint limit under dictionary size" do
+        let(:codepoint_limit) { dictionary_size / 2 }
 
         it "raises exception" do
           expect { subject.register }.to raise_error(/The incoming YAML document exceeds/)
@@ -261,8 +278,6 @@ describe LogStash::Filters::Translate do
       end
 
       context "dictionary is within limit" do
-        let(:dictionary_size) { 18 }
-
         it "returns the exact translation" do
           subject.register
           subject.filter(event)

data/spec/fixtures/dict.yml

@@ -1,3 +1,4 @@
 a : 1
 b : 2
 c : 3
+d : { "e": [1, "hello", true, "false", "1", "1.1"] }

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.4.3
+  version: 3.5.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-07-24 00:00:00.000000000 Z
+date: 2025-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -19,7 +19,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -31,12 +30,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-ecs_compatibility_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-  name: logstash-mixin-ecs_compatibility_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -45,12 +44,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-validator_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-validator_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -59,12 +58,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-deprecation_logger_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-deprecation_logger_support
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -73,12 +72,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-scheduler
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -87,12 +86,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: psych
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.1.0
-  name: psych
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -101,12 +100,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 5.1.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -115,12 +114,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-sequencing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-sequencing
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -129,12 +128,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec-wait
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec-wait
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -143,12 +142,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: benchmark-ips
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: benchmark-ips
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -177,6 +176,7 @@ files:
 - lib/logstash/filters/dictionary/file.rb
 - lib/logstash/filters/dictionary/json_file.rb
 - lib/logstash/filters/dictionary/memory.rb
+- lib/logstash/filters/dictionary/streaming_yaml_parser.rb
 - lib/logstash/filters/dictionary/yaml_file.rb
 - lib/logstash/filters/dictionary/yaml_visitor.rb
 - lib/logstash/filters/fetch_strategy/file.rb
@@ -205,7 +205,6 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -220,8 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Replaces field contents based on a hash or YAML file
 test_files: