logstash-filter-pilar 0.1.0

data/lib/logstash/filters/preprocessor.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'logstash/filters/parser'
+
+# The Preprocessor class is designed for processing and masking log events.
+# This class provides functionality to parse, anonymize, and sanitize log data,
+# ensuring sensitive information is masked before further processing or storage.
+#
+# Key Features:
+# - Initialization with dictionaries and regex patterns for custom preprocessing.
+# - Support for custom log formats using a flexible regex generator method.
+#
+# Usage:
+# The class is initialized with a gram dictionary for tokenizing log events, a set of regexes
+# for custom masking tailored to specific log files, and a log format for parsing log events.
+# Once initialized, it can generate regex patterns based on the provided log format and mask
+# sensitive information in log events, replacing it with a generic mask string.
+#
+# Methods:
+# - initialize(gram_dict, regexes, logformat): Sets up the preprocessing environment with the necessary dictionaries
+#   and formats.
+# - regex_generator(logformat): Generates a regular expression based on a specified log format, useful for parsing logs
+#   with known structures.
+# - token_splitter(log_line): splits a log line into tokens
+# - upload_grams_to_gram_dict(tokens): uploads a list of tokens into the single_gram, bi_gram and tri_gram dictionaries
+# - process_log_event(event): processes an entire log event by calling Parser.parse()
+#
+# Example:
+#   preprocessor = Preprocessor.new(gram_dict, logformat, content_specifier, regexes)
+#
+# This class is essential for log management systems where data privacy and security are paramount.
+class Preprocessor
+  def initialize(gram_dict, logformat, content_specifier, regexes)
+    # gram_dict for uploading log event tokens
+    @gram_dict = gram_dict
+
+    # Regex for specific log event format
+    @format = regex_generator(logformat)
+
+    # This is the content specifier in the @format regex
+    @content_specifier = content_specifier
+
+    @general_regex = [
+      /([\w-]+\.)+[\w-]+(:\d+)/, # url
+      %r{/?([0-9]+\.){3}[0-9]+(:[0-9]+)?(:|)}, # IP
+      /(?<=\W)(-?\+?\d+)(?=\W)|[0-9]+$/ # Numbers
+    ]
+
+    @regexes = regexes
+  end
+
+  # Method: regex_generator
+  # This method generates a regular expression based on a specified log format.
+  # It is designed to parse log files where the format of the logs is known and can be described using placeholders.
+  #
+  # Parameters:
+  # logformat: A string representing the log format.
+  #
+  # Returns:
+  # A Regexp object that can be used to match and extract data from log lines that follow the specified format.
+  def regex_generator(logformat)
+    # Split the logformat string into an array of strings and placeholders.
+    # Placeholders are identified as text within angle brackets (< >).
+    splitters = logformat.split(/(<[^<>]+>)/)
+
+    format = ''
+
+    # Iterate through the array of strings and placeholders.
+    splitters.each_with_index do |splitter, k|
+      if k.even?
+        # For the actual string parts (even-indexed elements),
+        # substitute spaces with the regex pattern for whitespace (\s+).
+        format += splitter.gsub(/\s+/, '\s+')
+      else
+        # For placeholders (odd-indexed elements),
+        # remove angle brackets and create named capture groups.
+        # This transforms each placeholder into a regex pattern that matches any characters.
+        header = splitter.gsub(/[<>]/, '')
+        format += "(?<#{header}>.*?)"
+      end
+    end
+
+    # Compile the complete regex pattern, anchored at the start and end,
+    Regexp.new("^#{format}$")
+  end
+
+  # Processes a log line to replace known dynamic tokens using the passed in regexes and the general regexes
+  #
+  # Parameters:
+  # log_line [String] the log line to be processed
+  # Returns:
+  # [String] a string that is a copy of the log except that the known dynamic tokens have been replaced with '<*>'
+  def preprocess_known_dynamic_tokens(log_line, regexes)
+    preprocessed_dynamic_token = {}
+    log_line = " #{log_line}"
+
+    regexes.each do |regex|
+      log_line.gsub!(regex).each_with_index do |match, index|
+        key = "manual_processed_dynamic_token_#{index + 1}"
+        preprocessed_dynamic_token[key] = match
+        '<*>'
+      end
+    end
+
+    @general_regex.each do |regex|
+      log_line.gsub!(regex).each_with_index do |match, index|
+        key = "global_processed_dynamic_token_#{index + 1}"
+        preprocessed_dynamic_token[key] = match
+        '<*>'
+      end
+    end
+    [log_line, preprocessed_dynamic_token]
+  end
+
+  # Splits a log line into tokens based on a given format and regular expression.
+  #
+  # Parameters:
+  # log_line [String] the log line to be processed
+  # Returns:
+  # [Array, nil] an array of tokens if matches are found, otherwise nil
+  def token_splitter(log_line)
+    # Finds matches in the stripped line for the regex format
+    stripped_log_line = log_line.strip
+    match = stripped_log_line.match(@format)
+
+    # If not match found, return nil
+    if match.nil?
+      [nil, nil]
+    else
+      # Gets content and return
+      content = match[@content_specifier]
+      line, preprocessed_dynamic_token = preprocess_known_dynamic_tokens(content, @regexes)
+      [line.strip.split, preprocessed_dynamic_token]
+    end
+  end
+
+  # Processes a given log event by tokenizing it, parsing it, and updating the gram dictionary.
+  #
+  # This method first calls the `token_splitter` method to split the log event into tokens based on the
+  # pre-configured format.
+  # The tokens are then passed to the `upload_grams` method, which iteratively uploads single grams,
+  # digrams, and trigrams to the `@gram_dict`.
+  #
+  # The process involves two primary steps: tokenization and dictionary updating.
+  # Tokenization is done based on the log format and involves masking sensitive information before splitting.
+  # Each token, digram, and trigram found in the log event is then uploaded to the gram dictionary, enhancing the
+  # dictionary's ability to process future log events.
+  #
+  # Parameters:
+  # log_event [String] the log event to be processed
+  # dynamic_token_threshold [Float] the threshold for a token to be considered a dynamic token or not
+  # parse [Boolean] a boolean that controls whether the log_event should be parsed. This will be set to False for
+  #                 seed log events.
+  #
+  # Returns:
+  # event_string [String], template_string[String], which are useful for log analysis and pattern recognition.
+  # It also updates the gram dict based on this information.
+  def process_log_event(log_event, dynamic_token_threshold, parse)
+    template_string = nil
+    dynamic_tokens = nil
+    all_dynamic_tokens = {}
+
+    # Split log event into tokens
+    tokens, preprocessed_dynamic_token = token_splitter(log_event)
+    all_dynamic_tokens.merge(preprocessed_dynamic_token) if preprocessed_dynamic_token
+
+    # If no tokens were returned, do not parse the logs and return
+    return if tokens.nil?
+
+    # Parse the log based on the pre-existing gramdict data
+    if parse
+      # Parse the log based on the pre-existing gramdict data
+      parser = Parser.new(@gram_dict, dynamic_token_threshold)
+      template_string, dynamic_tokens = parser.parse(tokens)
+
+      # there should be no conflicts here as long as all preprocess_known_dynamic_tokens have
+      # the format "[global/manual]_preprocessed_dynamic_token_{i}" and all the dynamic tokens have the
+      # format "dynamic_token_{i}"
+      all_dynamic_tokens.merge(dynamic_tokens) if dynamic_tokens
+    end
+
+    # Update gram_dict
+    @gram_dict.upload_grams(tokens)
+
+    [template_string, all_dynamic_tokens]
+  end
+end

data/logstash-filter-pilar.gemspec

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+  s.name          = 'logstash-filter-pilar'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Logstash Filter Plugin for Pilar'
+  s.description   = 'A plugin for parsing log events using PILAR'
+  s.homepage      = ''
+  s.authors       = %w[aaronabraham311 ZhangCreations]
+  s.email         = 'aaronabraham311@gmail.com'
+  s.require_paths = ['lib']
+  s.required_ruby_version = '>= 2.7.0'
+
+  # Files
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE',
+                'NOTICE.TXT']
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter',
+                 'rubygems_mfa_required' => 'true' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/filters/gramdict_spec.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require_relative '../spec_helper'
+require 'logstash/filters/gramdict'
+
+describe GramDict do
+  let(:logformat) { '<date> <time> <message>' }
+  let(:max_gram_dict_size) { 10_000 }
+
+  subject { GramDict.new(max_gram_dict_size) }
+
+  describe '#single_gram_upload' do
+    let(:gram) { 'example' }
+
+    it 'correctly updates the single gram count' do
+      expect { subject.single_gram_upload(gram) }
+        .to change { subject.instance_variable_get(:@single_gram_dict)[gram] }.from(nil).to(1)
+
+      expect { subject.single_gram_upload(gram) }
+        .to change { subject.instance_variable_get(:@single_gram_dict)[gram] }.from(1).to(2)
+    end
+  end
+
+  describe '#double_gram_upload' do
+    let(:double_gram) { 'example gram' }
+
+    it 'correctly updates the double gram count' do
+      expect { subject.double_gram_upload(double_gram) }
+        .to change { subject.instance_variable_get(:@double_gram_dict)[double_gram] }.from(nil).to(1)
+
+      expect { subject.double_gram_upload(double_gram) }
+        .to change { subject.instance_variable_get(:@double_gram_dict)[double_gram] }.from(1).to(2)
+    end
+  end
+
+  describe '#tri_gram_upload' do
+    let(:tri_gram) { 'example tri gram' }
+
+    it 'correctly updates the tri gram count' do
+      expect { subject.tri_gram_upload(tri_gram) }
+        .to change { subject.instance_variable_get(:@tri_gram_dict)[tri_gram] }.from(nil).to(1)
+
+      expect { subject.tri_gram_upload(tri_gram) }
+        .to change { subject.instance_variable_get(:@tri_gram_dict)[tri_gram] }.from(1).to(2)
+    end
+  end
+
+  describe '#upload_grams' do
+    context 'with one token' do
+      let(:tokens) { ['token1'] }
+
+      it 'updates only the single gram dictionary' do
+        expect { subject.upload_grams(tokens) }
+          .to change { subject.single_gram_dict['token1'] }.from(nil).to(1)
+        expect(subject.double_gram_dict.count).to eq(0)
+        expect(subject.tri_gram_dict.count).to eq(0)
+      end
+    end
+
+    context 'with two tokens' do
+      let(:tokens) { %w[token1 token2] }
+      let(:double_gram) { 'token1^token2' }
+
+      it 'updates the single and double gram dictionaries' do
+        expect { subject.upload_grams(tokens) }
+          .to change { subject.single_gram_dict['token1'] }.from(nil).to(1)
+          .and change { subject.single_gram_dict['token2'] }.from(nil).to(1)
+          .and change { subject.double_gram_dict[double_gram] }.from(nil).to(1)
+        expect(subject.tri_gram_dict.count).to eq(0)
+      end
+    end
+
+    context 'with three tokens' do
+      let(:tokens) { %w[token1 token2 token3] }
+      let(:double_gram1) { 'token1^token2' }
+      let(:double_gram2) { 'token2^token3' }
+      let(:tri_gram) { 'token1^token2^token3' }
+
+      it 'updates the single, double, and triple gram dictionaries' do
+        expect { subject.upload_grams(tokens) }
+          .to change { subject.single_gram_dict['token1'] }.from(nil).to(1)
+          .and change { subject.single_gram_dict['token2'] }.from(nil).to(1)
+          .and change { subject.single_gram_dict['token3'] }.from(nil).to(1)
+          .and change { subject.double_gram_dict[double_gram1] }.from(nil).to(1)
+          .and change { subject.double_gram_dict[double_gram2] }.from(nil).to(1)
+          .and change { subject.tri_gram_dict[tri_gram] }.from(nil).to(1)
+      end
+    end
+
+    context 'with an empty token array' do
+      let(:tokens) { [] }
+
+      it 'does not update any gram dictionaries' do
+        expect { subject.upload_grams(tokens) }
+          .not_to(change { subject.single_gram_dict })
+
+        expect { subject.upload_grams(tokens) }
+          .not_to(change { subject.double_gram_dict })
+
+        expect { subject.upload_grams(tokens) }
+          .not_to(change { subject.tri_gram_dict })
+      end
+    end
+  end
+end

data/spec/filters/parser_spec.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require_relative '../spec_helper'
+require 'logstash/filters/parser'
+require 'logstash/filters/gramdict'
+
+describe Parser do
+  let(:tokens_list) { [%w[token1a token1b], %w[token2a token2b token2c], %w[token3a token3b]] }
+  let(:threshold)   { 0.5 }
+
+  # Create an instance of GramDict
+  let(:gramdict) do
+    gd = GramDict.new(10_000)
+
+    # Manually setting the dictionaries
+    gd.instance_variable_set(:@single_gram_dict, { 'token2a' => 2, 'key2' => 2 })
+    gd.instance_variable_set(:@double_gram_dict, { 'token2a^token2b' => 2, 'token2b' => 4 })
+    gd.instance_variable_set(:@tri_gram_dict,    { 'token2a^token2b^token2c' => 5, 'key2' => 6 })
+
+    gd
+  end
+
+  # Create an instance of Parser
+  subject(:parser) { Parser.new(gramdict, threshold) }
+
+  describe '#initialize' do
+    it 'initializes with the correct attributes' do
+      expect(parser.instance_variable_get(:@gramdict)).to eq(gramdict)
+      expect(parser.instance_variable_get(:@threshold)).to eq(threshold)
+    end
+  end
+
+  describe '#dynamic_token?' do
+    let(:dynamic_index) { [] }
+
+    context 'when the token index is zero' do
+      it 'identifies the token as dynamic' do
+        tokens = tokens_list.first
+        index = 0
+        expect(parser.dynamic_token?(tokens, dynamic_index, index)).to be false
+      end
+    end
+
+    context 'when the token index is one and does not meet dynamic criteria' do
+      it 'identifies the token as not dynamic' do
+        tokens = tokens_list[1]
+        index = 1
+        expect(parser.dynamic_token?(tokens, dynamic_index, index)).to be false
+      end
+    end
+
+    context 'when the token index is greater than one and meets dynamic criteria' do
+      it 'identifies the token as dynamic' do
+        tokens = tokens_list[1]
+        index = 2
+        expect(parser.dynamic_token?(tokens, dynamic_index, index)).to be false
+      end
+    end
+  end
+
+  describe '#find_dynamic_indices' do
+    it 'returns the correct dynamic index for a given tokens array' do
+      tokens = tokens_list[2]
+
+      dynamic_indices = parser.find_dynamic_indices(tokens)
+
+      expected_indices = [1]
+
+      expect(dynamic_indices).to eq(expected_indices)
+    end
+  end
+
+  describe '#template_generator' do
+    it 'generates the correct template based on dynamic indices' do
+      tokens = tokens_list[1]
+      dynamic_indices = [1]
+
+      template, dynamic_tokens = parser.template_generator(tokens, dynamic_indices)
+
+      # template = template_generator_return_value[0]
+      # dynamic_tokens = template_generator_return_value[1]
+
+      expected_template = 'token2a <*> token2c '
+      expected_dynamic_tokens = { 'dynamic_token_1' => 'token2b' }
+
+      expect(template).to eq(expected_template)
+      expect(dynamic_tokens).to eq(expected_dynamic_tokens)
+    end
+  end
+
+  describe '#parse' do
+    it 'parses the tokens list and generates strings in the correct format' do
+      tokens = tokens_list[1]
+      template_string, dynamic_tokens = parser.parse(tokens)
+
+      expected_template_string = 'token2a token2b token2c '
+      expected_dynamic_tokens = {}
+
+      expect(template_string).to eq(expected_template_string)
+      expect(dynamic_tokens).to eq(expected_dynamic_tokens)
+    end
+  end
+end

data/spec/filters/pilar_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../spec_helper'
+require 'logstash/filters/pilar'
+
+describe LogStash::Filters::Pilar do
+  let(:config) { { 'source_field' => 'sample_log', 'dynamic_token_threshold' => 0.5 } }
+  subject(:pilar_filter) { described_class.new(config) }
+
+  before do
+    pilar_filter.register
+  end
+
+  describe 'registration' do
+    it 'correctly register without errors' do
+      expect { pilar_filter }.not_to raise_error
+    end
+  end
+
+  describe 'filtering' do
+    sample_log = '- 1120928280 2005.07.09 R21-M0-NB-C:J05-U11 2005-07-09-09.58.00.188544 R21-M0-NB-C:J05-U11 ' \
+                 'RAS KERNEL INFO generating core.10299'
+
+    let(:event) { LogStash::Event.new('sample_log' => sample_log) }
+
+    before do
+      pilar_filter.filter(event)
+    end
+
+    it 'correctly sets the dynamic_tokens field' do
+      expect(event.get('dynamic_tokens')).not_to be_nil
+    end
+
+    it 'correctly sets the template_string field' do
+      expect(event.get('template_string')).not_to be_nil
+    end
+
+    it 'correctly sets the raw_log field to the value of the source_field' do
+      expect(event.get('raw_log')).to eq(sample_log)
+    end
+  end
+end

data/spec/filters/preprocessor_spec.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require_relative '../spec_helper'
+require 'logstash/filters/preprocessor'
+require 'logstash/filters/gramdict'
+
+describe Preprocessor do
+  let(:gram_dict) { GramDict.new(10_000) }
+  let(:logformat) { '<date> <time> <message>' }
+  let(:content_specifier) { 'message' }
+  let(:dynamic_token_threshold) { 0.5 }
+  let(:regexes) { ["(\d+.){3}\d+"] }
+  let(:preprocessor) { Preprocessor.new(gram_dict, logformat, content_specifier, regexes) }
+
+  describe '#regex_generator' do
+    it 'generates a regex based on log format' do
+      logformat = '<date> <time> <message>'
+      regex = preprocessor.send(:regex_generator, logformat)
+      expect(regex).to be_a(Regexp)
+      expect('2023-01-01 10:00:00 Sample Log Message').to match(regex)
+    end
+  end
+
+  describe '#preprocess_known_dynamic_tokens' do
+    let(:log_line) { 'User logged in from IP 192.168.1.1' }
+    let(:regexes) { [/User/] }
+
+    it 'returns processed log line and dynamic tokens dictionary' do
+      processed_log, dynamic_tokens = preprocessor.preprocess_known_dynamic_tokens(log_line, regexes)
+      expect(processed_log).not_to include('User')
+      expect(processed_log).to include('<*>')
+      expect(dynamic_tokens).to be_a(Hash)
+      expect(dynamic_tokens.keys).to include('global_processed_dynamic_token_1')
+    end
+
+    context 'with general regexes applied' do
+      it 'replaces both specific and general dynamic tokens with "<*>"' do
+        processed_log, = preprocessor.preprocess_known_dynamic_tokens(log_line, regexes)
+        expect(processed_log).not_to include('192.168.1.1')
+        expect(processed_log).not_to include('User')
+        expect(processed_log).to include('<*>').twice
+      end
+    end
+
+    context 'when extracting dynamic tokens' do
+      it 'correctly extracts and stores dynamic tokens with indices' do
+        _, dynamic_tokens = preprocessor.preprocess_known_dynamic_tokens(log_line, [/user/i])
+        expect(dynamic_tokens['manual_processed_dynamic_token_1']).to eq('User')
+      end
+    end
+
+    context 'when no matching tokens are found' do
+      let(:unmatched_log_line) { 'Static log message without dynamic content' }
+
+      it 'returns the log line unchanged and an empty dynamic tokens dictionary' do
+        processed_log, dynamic_tokens = preprocessor.preprocess_known_dynamic_tokens(unmatched_log_line, regexes)
+        expect(processed_log).to eq(" #{unmatched_log_line}")
+        expect(dynamic_tokens).to be_empty
+      end
+    end
+  end
+
+  describe '#token_splitter' do
+    it 'splits a log line into tokens when a match is found' do
+      log_line = '2023-01-01 10:00:00 Sample Log Message'
+      tokens = preprocessor.token_splitter(log_line)
+      expect(tokens).to be_an(Array)
+      expect(tokens).to eq([%w[Sample Log Message], {}])
+    end
+
+    it 'returns nil when no match is found in the log line' do
+      log_line = ''
+      tokens = preprocessor.token_splitter(log_line)
+      expect(tokens).to eq([nil, nil])
+    end
+  end
+
+  describe '#process_log_event' do
+    let(:log_event) { '2023-01-01 10:00:00 Sample Log Event' }
+    let(:threshold) { 0.5 }
+
+    before do
+      allow(preprocessor).to receive(:token_splitter).and_call_original
+      allow(gram_dict).to receive(:upload_grams)
+      allow(gram_dict).to receive(:single_gram_upload)
+      allow(gram_dict).to receive(:double_gram_upload)
+      allow(gram_dict).to receive(:tri_gram_upload)
+    end
+
+    it 'calls token_splitter with the log event' do
+      preprocessor.process_log_event(log_event, dynamic_token_threshold, true)
+      expect(preprocessor).to have_received(:token_splitter).with(log_event)
+    end
+
+    context 'when tokens are extracted from log event' do
+      let(:tokens) { %w[Sample Log Event] }
+
+      before do
+        allow(preprocessor).to receive(:token_splitter).and_return([tokens, {}])
+        allow(gram_dict).to receive(:upload_grams)
+        preprocessor.process_log_event(log_event, dynamic_token_threshold, true)
+      end
+
+      it 'calls upload_grams with extracted tokens' do
+        expect(gram_dict).to have_received(:upload_grams)
+      end
+    end
+
+    context 'when no tokens are extracted from log event (token_splitter returns nil)' do
+      before do
+        allow(preprocessor).to receive(:token_splitter).and_return(nil)
+        allow(gram_dict).to receive(:upload_grams)
+        preprocessor.process_log_event(log_event, dynamic_token_threshold, true)
+      end
+
+      it 'does not call upload_grams' do
+        expect(gram_dict).not_to have_received(:upload_grams)
+      end
+    end
+
+    context 'when parse is set to false' do
+      before do
+        allow(Parser).to receive(:new).and_return(double('Parser', parse: nil))
+        preprocessor.process_log_event(log_event, dynamic_token_threshold, false)
+      end
+
+      it 'does not call parser.parse' do
+        expect(Parser).not_to have_received(:new)
+      end
+    end
+
+    context 'when parse is set to true' do
+      before do
+        allow(Parser).to receive(:new).and_return(double('Parser', parse: nil))
+        preprocessor.process_log_event(log_event, dynamic_token_threshold, true)
+      end
+
+      it 'does call parser.parse' do
+        expect(Parser).to have_received(:new)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'logstash/devutils/rspec/spec_helper'