logstash-filter-pilar 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0df55eb80e39d1eae306a3a06347ea7f3437d72ccb92beeb7488648a0a03f071
+  data.tar.gz: e640ff08c16ae6da6352832d8517a090e4c5c213c6a8baf29c05a44ac30d4b27
+SHA512:
+  metadata.gz: bd703671387aa1a876f554066bcbcddb718279bbd9aeb99a1cfc4323e7e4fc519ba1f2d3f43c9c9bb69ba0ccf0939527649e7fc60988ffb5faa13247c3cfa718
+  data.tar.gz: 39034b76f9dfd09844207f8405af99e893fba3b5d67c456c35d8ca77ffd6aefcb3347b71757303dbb3fd1db4a2eeda5efe6e8dead824d63a0a41337358246df7

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* aaronabraham311 - aaronabraham311@gmail.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-pilar
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec
+
+gem 'lru_redux'
+
+logstash_path = ENV.fetch('LOGSTASH_PATH', nil)
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', path: "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', path: "#{logstash_path}/logstash-core-plugin-api"
+end
+
+group :development do
+  gem 'execjs', require: false
+  gem 'pre-commit', require: false
+  gem 'rspec', '~> 3.12'
+  gem 'rubocop', require: false
+end

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,139 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Development and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Install logstash locally and add the env variable 'LOGSTASH_PATH' which points to your logstash instance, to your path.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+- Run `rubocop -A` to run and fix Ruby style guide issues
+
+- Add the following to `.git/hooks/pre-commit`
+
+```
+ #!/usr/bin/env sh
+
+ # This hook has a focus on portability.
+ # This hook will attempt to setup your environment before running checks.
+ #
+ # If you would like `pre-commit` to get out of your way and you are comfortable
+ # setting up your own environment, you can install the manual hook using:
+ #
+ #     pre-commit install --manual
+ #
+
+ # This is a work-around to get GitHub for Mac to be able to run `node` commands
+ # https://stackoverflow.com/questions/12881975/git-pre-commit-hook-failing-in-github-for-mac-works-on-command-line
+ PATH=$PATH:/usr/local/bin:/usr/local/sbin
+
+
+ cmd=`git config pre-commit.ruby 2>/dev/null`
+ if   test -n "${cmd}"
+ then true
+ elif which rvm   >/dev/null 2>/dev/null
+ then cmd="rvm default do ruby"
+ elif which rbenv >/dev/null 2>/dev/null
+ then cmd="rbenv exec ruby"
+ else cmd="ruby"
+ fi
+
+ export rvm_silence_path_mismatch_check_flag=1
+
+ ${cmd} -rrubygems -e '
+   begin
+     require "pre-commit"
+     true
+   rescue LoadError => e
+     $stderr.puts <<-MESSAGE
+ pre-commit: WARNING: Skipping checks because: #{e}
+ pre-commit: Did you set your Ruby version?
+ MESSAGE
+     false
+   end and PreCommit.run
+ '
+```
+
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-pilar", :path => "/your/local/logstash-filter-pilar"
+```
+- Install plugin
+```sh
+bin/logstash-plugin install --no-verify
+```
+- Run Logstash with your plugin, you can test your code by typing a log in the command line, and the output will immediately be reflected
+```sh
+bin/logstash -e 'filter {pilar {}}'
+```
+Alternatively, you can include a file path for seed logs by running the following:
+```sh
+bin/logstash -e 'filter { pilar { seed_logs_path => "example/file/path" } }'
+```
+
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-pilar.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/logstash-plugin install /your/local/plugin/logstash-filter-pilar.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/main/CONTRIBUTING.md) file.

data/lib/logstash/filters/gramdict.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'lru_redux'
+
+# The GramDict class is designed for processing and analyzing log events.
+# It creates dictionaries for single, double, triple, and four-word combinations
+# (n-grams) found in the log data. The class is initialized with several parameters:
+#
+# Methods:
+# - single_gram_upload(gram): Updates the count of a single word in the dictionary.
+# - double_gram_upload(gram): Updates the count of a double word combination in the dictionary.
+# - tri_gram_upload(gram): Updates the count of a triple word combination in the dictionary.
+# - four_gram_upload(gram): Updates the count of a four-word combination in the dictionary.
+# - Getters for each gram_dict (asides from four_gram_dict)
+#
+# This class is useful for log file analysis, especially for identifying common patterns
+# and anomalies in log entries.
+class GramDict
+  def initialize(max_gram_dict_size)
+    @max_gram_dict_size = max_gram_dict_size
+    @tri_gram_dict = LruRedux::Cache.new(max_gram_dict_size)
+    @double_gram_dict = LruRedux::Cache.new(max_gram_dict_size)
+    @single_gram_dict = LruRedux::Cache.new(max_gram_dict_size)
+  end
+
+  # Method: single_gram_upload
+  # This method updates the frequency count of a single gram (word or token) in a hash map.
+  # It increases the count of the gram if it already exists in the hash map,
+  # or initializes it to 1 if it's the first occurrence.
+  #
+  # Parameters:
+  # gram: A string representing the single gram whose count needs to be updated.
+  #
+  # Returns:
+  # Nothing. It updates the @single_gram_dict in place.
+  def single_gram_upload(gram)
+    if @single_gram_dict.key?(gram)
+      @single_gram_dict[gram] += 1
+    else
+      @single_gram_dict[gram] = 1
+    end
+  end
+
+  # Method: double_gram_upload
+  # This method is used to update the frequency count of a double gram (pair of words or tokens) in a hash map.
+  # It increments the count of the double gram if it exists,
+  # or initializes it to 1 if it's not already present.
+  #
+  # Parameters:
+  # gram: A string representing the double gram to be updated in the hash map.
+  #
+  # Returns:
+  # Nothing. It updates the @double_gram_dict in place.
+  def double_gram_upload(gram)
+    if @double_gram_dict.key?(gram)
+      @double_gram_dict[gram] += 1
+    else
+      @double_gram_dict[gram] = 1
+    end
+  end
+
+  # Method: tri_gram_upload
+  # This method updates the count of a tri gram (sequence of three words or tokens) in a hash map.
+  # It increases the count if the tri gram is already present,
+  # or sets it to 1 for a new tri gram.
+  #
+  # Parameters:
+  # gram: A string representing the tri gram for frequency updating.
+  #
+  # Returns:
+  # Nothing. It modifies the @tri_gram_dict internally.
+  def tri_gram_upload(gram)
+    if @tri_gram_dict.key?(gram)
+      @tri_gram_dict[gram] += 1
+    else
+      @tri_gram_dict[gram] = 1
+    end
+  end
+
+  # Method: single_gram_dict
+  # This method is a getter for the single_gram_dict
+  #
+  # Parameters:
+  # Nothing.
+  #
+  # Returns:
+  # The @single_gram_dict member
+  attr_reader :single_gram_dict
+
+  # Method: double_gram_dict
+  # This method is a getter for the double_gram_dict
+  #
+  # Parameters:
+  # Nothing.
+  #
+  # Returns:
+  # The @double_gram_dict member
+  attr_reader :double_gram_dict
+
+  # Method: tri_gram_dict
+  # This method is a getter for the tri_gram_dict
+  #
+  # Parameters:
+  # Nothing.
+  #
+  # Returns:
+  # The @tri_gram_dict member
+  attr_reader :tri_gram_dict
+
+  # Processes an array of tokens to upload single grams, digrams, and trigrams to the @gram_dict.
+  #
+  # This method iterates through each token in the array. For each token, it uploads the token as a single gram.
+  # Additionally, if the current token is not the first in the array, it creates and uploads a digram using the current
+  # and previous token.
+  # If the token is at least the third in the array, the method also creates and uploads a trigram using the current
+  # token and the two preceding it.
+  # The tokens in digrams and trigrams are separated by a defined separator (`token_seperator`).
+  #
+  # Parameters:
+  # tokens [Array<String>] an array of string tokens to be processed
+  #
+  # Returns:
+  # [void] this method does not return a value but updates single_gram_dict, double_gram_dict and tri_gram_dict
+  def upload_grams(tokens)
+    token_seperator = '^'
+
+    # Iterate across all tokens
+    tokens.each_with_index do |token, index|
+      # Upload single gram
+      single_gram_upload(token)
+
+      # If possible, upload a digram
+      if index.positive?
+        first_token = tokens[index - 1]
+        digram = first_token + token_seperator + token
+        double_gram_upload(digram)
+      end
+
+      # If possible, upload a trigram
+      next unless index > 1
+
+      first_token = tokens[index - 2]
+      second_token = tokens[index - 1]
+      trigram = first_token + token_seperator + second_token + token_seperator + token
+      tri_gram_upload(trigram)
+    end
+  end
+end

data/lib/logstash/filters/parser.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'logstash/filters/gramdict'
+
+# The Parser class is responsible for analyzing log tokens and generating templates and events.
+# It identifies dynamic tokens within logs and creates standardized templates
+# by replacing these dynamic tokens. The class is initialized with three parameters:
+# - gramdict: An instance of the GramDict class used for n-gram frequency analysis.
+# - threshold: A numeric value used to determine if a token is dynamic based on its frequency.
+# If it's frequency is less than this threshold, it's dynamic.
+#
+# Methods:
+# - dynamic_token?: Determines if a token is dynamic by comparing its frequency to the set threshold.
+# - calculate_token_frequency: Calculates frequency of a token considering its index position.
+# - calculate_bigram_frequency: Determines frequency based on adjacent tokens (bigrams).
+# - calculate_trigram_frequency: Calculates frequency based on trigram context.
+# - find_dynamic_indices: Identifies all dynamic tokens in a log entry.
+# - template_generator: Generates a log template by replacing dynamic tokens.
+# - parse: Processes each token list to generate event strings and templates.
+class Parser
+  def initialize(gramdict, threshold)
+    @gramdict = gramdict
+    @threshold = threshold
+  end
+
+  # Method: dynamic_token?
+  # This method evaluates if a given token in a log is dynamic by assessing its frequency relative to a set threshold.
+  # A token is deemed dynamic if its frequency is equal to or lower than the threshold value.
+  #
+  # Parameters:
+  # - tokens: An array of tokens from a log entry.
+  # - dynamic_indices: An array containing indices of previously identified dynamic tokens.
+  # - index: The index of the current token being evaluated.
+  #
+  # Returns:
+  # A boolean indicating whether the token is dynamic (true) or static (false).
+  def dynamic_token?(tokens, dynamic_indices, index)
+    frequency = calculate_token_frequency(tokens, dynamic_indices, index)
+    frequency <= @threshold
+  end
+
+  # Method: calculate_token_frequency
+  # This method determines the frequency of a token within a log entry, considering the context provided by adjacent
+  # tokens.
+  # It switches between bigram and trigram frequency calculations based on the token's position and the dynamic status
+  # of preceding tokens.
+  #
+  # The method returns 1 for the first token (index 0), giving it maximum frequency as its assuming no previous context.
+  # For the second token (index 1), it calculates the bigram frequency. For a token where the token two indices before
+  # is dynamic, a bigram is also used as trigram frequency calculation does not make sense on a dynamic token.
+  # In all other cases, it calculates the trigram frequency.
+  #
+  # Parameters:
+  # - tokens: An array of tokens from the log entry.
+  # - dynamic_indices: An array of indices for previously identified dynamic tokens.
+  # - index: The index of the current token for which the frequency is calculated.
+  #
+  # Returns:
+  # The calculated frequency of the token as a float, based on bigram or trigram analysis.
+  def calculate_token_frequency(tokens, dynamic_indices, index)
+    if index.zero?
+      1
+    elsif index == 1 || dynamic_indices.include?(index - 2)
+      calculate_bigram_frequency(tokens, index)
+    else
+      calculate_trigram_frequency(tokens, index)
+    end
+  end
+
+  # Method: calculate_bigram_frequency
+  # This method calculates the frequency of a token within the context of a bigram (pair of adjacent tokens).
+  # It forms a bigram with the token and its preceding token, then checks their frequency in the GramDict instance.
+  # The frequency is determined as the ratio of the bigram frequency to the frequency of the preceding single token.
+  #
+  # Parameters:
+  # tokens: An array of tokens representing the log entry.
+  # index: The current index of the token for which the bigram frequency is being calculated.
+  #
+  # Returns:
+  # The frequency of the bigram as a float. If the bigram or singlegram is not found in the dictionaries,
+  # it returns 0, indicating a lack of previous occurrences.
+  def calculate_bigram_frequency(tokens, index)
+    singlegram = tokens[index - 1]
+    doublegram = "#{singlegram}^#{tokens[index]}"
+
+    if @gramdict.double_gram_dict.key?(doublegram) && @gramdict.single_gram_dict.key?(singlegram)
+      @gramdict.double_gram_dict[doublegram].to_f / @gramdict.single_gram_dict[singlegram]
+    else
+      0
+    end
+  end
+
+  # Method: calculate_trigram_frequency
+  # This method calculates the frequency of a token within the context of a trigram (sequence of three adjacent tokens).
+  # It forms a trigram with the token and its two preceding tokens and also considers the intermediate bigram.
+  # The frequency is determined as the ratio of the trigram frequency to the frequency of the preceding bigram.
+  #
+  # Parameters:
+  # tokens: An array of tokens representing the log entry.
+  # index: The current index of the token for which the trigram frequency is being calculated.
+  #
+  # Returns:
+  # The frequency of the trigram as a float. If the trigram or the intermediate bigram is not found in the dictionaries,
+  # it returns 0, suggesting a unique or rare occurrence in the logs.
+  def calculate_trigram_frequency(tokens, index)
+    doublegram = "#{tokens[index - 2]}^#{tokens[index - 1]}"
+    trigram = "#{doublegram}^#{tokens[index]}"
+
+    if @gramdict.tri_gram_dict.key?(trigram) && @gramdict.double_gram_dict.key?(doublegram)
+      @gramdict.tri_gram_dict[trigram].to_f / @gramdict.double_gram_dict[doublegram]
+    else
+      0
+    end
+  end
+
+  # Method: find_dynamic_indices
+  # This method identifies dynamic tokens in a given log entry. It iterates through the tokens
+  # and uses the dynamic_token? method to check if each token is dynamic. Dynamic tokens are those
+  # whose frequency is less than or equal to a certain threshold, suggesting variability in log entries.
+  #
+  # Parameters:
+  # tokens: An array of tokens representing the log entry.
+  #
+  # Returns:
+  # An array of indices corresponding to dynamic tokens within the log entry.
+  def find_dynamic_indices(tokens)
+    dynamic_indices = []
+    if tokens.length >= 2
+      index = 1
+      while index < tokens.length
+        dynamic_indices << index if dynamic_token?(tokens, dynamic_indices, index) # Directly calling dynamic_token?
+        index += 1
+      end
+    end
+    dynamic_indices
+  end
+
+  # Method: template_generator
+  # Generates a standardized log template from a list of tokens. This method replaces dynamic tokens
+  # (identified by their indices in dynamic_indices) with a placeholder symbol '<*>' and stores the tokens
+  # for output. The result is a template that represents the static structure of the log entry,
+  # with dynamic parts parsed out.
+  #
+  # Parameters:
+  # tokens: An array of tokens from the log entry.
+  # dynamic_indices: An array of indices indicating which tokens are dynamic.
+  #
+  # Returns:
+  # template: A string representing the log template, with dynamic tokens replaced by '<*>'.
+  # dynamic_tokens: a map of dynamic tokens, structured as { "dynamic_token_{index}" : <dynamic_token> }
+  def template_generator(tokens, dynamic_indices)
+    template = String.new('')
+    dynamic_tokens = {}
+
+    tokens.each_with_index do |token, index|
+      if dynamic_indices.include?(index)
+        template << '<*> '
+        dynamic_tokens["dynamic_token_#{index}"] = token
+      else
+        template << "#{token} "
+      end
+    end
+
+    [template, dynamic_tokens]
+  end
+
+  # Method: parse
+  # This method processes the log entry represented as tokens. It identifies dynamic tokens,
+  # generates a log template, and then compiles two strings: event_string and template_string.
+  # The event_string maps each event to its template, while template_string counts the occurrences
+  # of each template. It also ensures that templates are properly formatted by removing certain characters.
+
+  # Parameters:
+  # log_tokens: An array of tokens from the log entry.
+  # Returns:
+  # An array containing the event_string and template_string, which are useful for log analysis and pattern recognition.
+  def parse(log_tokens)
+    dynamic_indices = find_dynamic_indices(log_tokens)
+    template_string, dynamic_tokens = template_generator(log_tokens, dynamic_indices)
+
+    # TODO: The Python iteration of the parser does a few regex checks here on the templates
+    # It's unclear based on prelimilarly data if we need this, but once the full plugin has been fleshed out we can
+    # revisit
+    template_string.gsub!(/[,'"]/, '')
+
+    [template_string, dynamic_tokens]
+  end
+end

data/lib/logstash/filters/pilar.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'logstash/filters/base'
+require 'logstash/filters/gramdict'
+require 'logstash/filters/preprocessor'
+
+module LogStash
+  module Filters
+    # Parses log events using PILAR
+    class Pilar < LogStash::Filters::Base
+      config_name 'pilar'
+
+      # Optional configuration: Specify the field name that contains the message to be used.
+      # If this is not set, the filter will use the value of the "message" field by default.
+      config :source_field, validate: :string, default: 'message'
+
+      # To improve accuracy of the parsing plugin, users will have the option of sending pre-existing logs
+      # which the parser will use to seed data structures. This seeding process will greatly improve accuracy
+      # of subsequent log parsing
+      config :seed_logs_path, validate: :path, required: false
+
+      # The parsing algorithm requires a numeric probabilistic threshold to determine whether a
+      # particular parsed token is a dynamic token (i.e. changes extremely frequently) or if it is static.
+      # If the probability that the token is a dynamic token is above this threshold, the token is considered
+      # dynamic. The default threshold is set at 0.5. Since this is a probability threshold, the config value
+      # must be between 0 and 1.
+      config :dynamic_token_threshold, validate: :number, required: false, default: 0.5
+
+      # The standard log format for the application must be included in this plugin's configuration in the format
+      # of "<log_part_1_placeholder> <log_part_2_placeholder> ...". For example, if logs are usually of the form
+      # "02012024 1706542368 Random log", then the log format would be "<date> <time> <message>".
+      # If no log format is included, we will use the default of "<date> <time> <message>"
+      config :logformat, validate: :string, required: false, default: '<date> <time> <message>'
+
+      # The content_specifier variable is the placeholder value in the `logformat` variable which the parser should use
+      # to identify the actual log message. For example, if `logformat = '<date> <time> <message>'`, then the
+      # content_specifier should be 'message' since this is the part of the log that the parser should parse. The
+      # default will be 'message', matching the default format in the `logformat` variable
+      config :content_specifier, validate: :string, required: false, default: 'message'
+
+      # The regex is an array of strings that will be converted to regexes that the user can input in order to supply
+      # the parser with prelimiary information about what is a a dynamic component of the log. For example, if the
+      # user wants to demark that IP addresses are known dynamic tokens, then the user can pass in passes in
+      # ['(\d+\.){3}\d+'] for IP addresses to be extracted before parsing begins.
+      config :regexes, validate: :string, list: true, required: false, default: []
+
+      # This determines the maximum size of the single, double, and triple gram dictionaries respectively.
+      # Upon any of those hash maps reaching their maximum size, a LRU evicition policy is used to remove items.
+      # This controls the upper limit of the memory usage of this filter.
+      config :maximum_gram_dict_size, validate: :number, required: false, default: 10_000
+
+      def register
+        @linenumber = 1
+        @regexes = regexes.map { |regex| Regexp.new(regex) }
+
+        # Check if dynamic_token_threshold is between 0 and 1
+        return unless @dynamic_token_threshold < 0.0 || @dynamic_token_threshold > 1.0
+
+        raise LogStash::ConfigurationError, 'dynamic_token_threshold must be between 0 and 1'
+      end
+
+      def filter(event)
+        # Initialize gramdict and preprocessor for this thread if not already done
+        unless Thread.current[:gramdict] && Thread.current[:preprocessor]
+          Thread.current[:gramdict] = GramDict.new(@maximum_gram_dict_size)
+          Thread.current[:preprocessor] =
+            Preprocessor.new(Thread.current[:gramdict], @logformat, @content_specifier, @regexes)
+
+          # Populate gramdict with seed logs
+          if @seed_logs_path && ::File.exist?(@seed_logs_path)
+            ::File.open(@seed_logs_path, 'r') do |seed_logs|
+              seed_logs.each_line do |seed_log|
+                Thread.current[:preprocessor].process_log_event(seed_log, @dynamic_token_threshold, false)
+              end
+            end
+          end
+        end
+
+        # Use the message from the specified source field
+        if event.get(@source_field)
+
+          processed_log = Thread.current[:preprocessor].process_log_event(
+            event.get(@source_field), @dynamic_token_threshold, true
+          )
+
+          if processed_log
+            template_string, dynamic_tokens = processed_log
+
+            # Set the new values in the returned event
+            event.set('template_string', template_string)
+            event.set('dynamic_tokens', dynamic_tokens)
+          else
+            event.set('dynamic_tokens', nil)
+            event.set('template_string', nil)
+          end
+
+          # include the raw log message
+          raw_log = event.get(@source_field)
+          event.set('raw_log', raw_log.strip)
+        end
+
+        # Emit event
+        filter_matched(event)
+      end
+    end
+  end
+end