logstash-output-site24x7 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bd8186dfce378fb29ed98539b61fde9687671c20b3c0d67e4595fa5c7f5bb04
-  data.tar.gz: c331664bfbec18153a6bad79233541df5e13e06fc38e112f1a3434aff2e1f225
+  metadata.gz: f50f2f0c133e264a4bc19d63275f26bb4f80563aa8fd65c6b4da22159e8b5c26
+  data.tar.gz: 1117f9c487cdf42c1e0be5ae2e01a173230d3ae0936538b8af627b48283caa66
 SHA512:
-  metadata.gz: 8304ad22beade27cfd7b5490428e7a9d941df97e925596de880edd1ae6f606b3a281e66e7758b216925962f009c6a140cdb7825837ed45ef31d25ed1dadba668
-  data.tar.gz: f1d6c23a49abc27ab5d1545e6a19fa1cdc91840e70eb5384613280c52fbe69c423d578dea18f9e6bf77e32bb227b06bb5d8fd6e4a1dd678ef5d59c2b70ba4b89
+  metadata.gz: ea1bf33015d904fc2f11e4ab0275e795b2f7c6695f339abd7a101aead93bb2979dfca99ed751b233301e057b8a89d2e0bceef86fb03143d2947a641a5c677f4b
+  data.tar.gz: 75804b58617818495b5a2951fb3efcdfd5483ad80bfac78f111d2e121cf418be993f5b457953f99f8948ef27527c6170485848c891dff6710cebb22c884cb736

data/CHANGELOG.md

@@ -1,2 +1,4 @@
 ## 0.1.0
   - Created output plugin to send logs to Site24x7
+## 0.1.1
+  - log_source parameter support added

data/README.md

@@ -29,6 +29,7 @@ Property | Description | Default Value
 log_type_config | log_type_config of your configured log type in site24x7 | nil
 max_retry | Number of times to resend failed uploads | 3
 retry_interval |  Time interval to sleep initially between retries, exponential step-off | 2 seconds
+log_source | Define custom values, such as the name of the server or environment from where the logs originate | Hostname 
 ## Need Help?
 
 If you need any support please contact us at support@site24x7.com.

data/lib/logstash/outputs/site24x7.rb

@@ -4,6 +4,7 @@ require "zlib"
 require "date"
 require "json"
 require "base64"
+require "digest"
 
 class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
   config_name "site24x7"
@@ -17,6 +18,7 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
   default :codec, "json"
 
   config :log_type_config,:validate => :string, :required => true
+  config :log_source,:validate => :string, :required => false, :default => Socket.gethostname
   config :max_retry, :validate => :number, :required => false,  :default => 3
   config :retry_interval, :validate => :number, :required => false, :default => 2
 	
@@ -46,8 +48,81 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
     @logtype_config = JSON.parse(base64_url_decode(@log_type_config))
     @s247_custom_regex = if @logtype_config.has_key? 'regex' then Regexp.compile(@logtype_config['regex'].gsub('?P<','?<')) else nil end
     @s247_ignored_fields = if @logtype_config.has_key? 'ignored_fields' then @logtype_config['ignored_fields'] else [] end
+    @datetime_regex = if @logtype_config.has_key?'dateRegex' then Regexp.compile(@logtype_config['dateRegex'].gsub('?P<','?<')) else nil end
+
+    @ml_regex = if @logtype_config.has_key? 'ml_regex' then Regexp.compile(@logtype_config['ml_regex'].gsub('?P<','?<')) else nil end
+    @ml_end_regex = if @logtype_config.has_key? 'ml_end_regex' then Regexp.compile(@logtype_config['ml_end_regex'].gsub('?P<','?<')) else nil end
+    @max_ml_count = if @logtype_config.has_key? 'ml_regex' then @s247_custom_regex.inspect.scan('\<NewLine\>').length else nil end
+    @max_trace_line = 100
+    @ml_trace = ''
+    @ml_trace_buffer = ''
+    @ml_found = false
+    @ml_end_line_found = false
+    @ml_data = nil
+    @ml_count = 0
+
+    @json_data = ''
+    @sub_pattern = {}
+
+    if !(@logtype_config.has_key?('jsonPath'))
+      @message_key = get_last_group_inregex(@s247_custom_regex)
+    end
+    
+    if @logtype_config.has_key?('jsonPath')
+      @logtype_config['jsonPath'].each_with_index do | key, index |
+        if key.has_key?('pattern')
+          begin
+            if Regexp.new(key['pattern'].gsub('?P<','?<'))
+              @sub_pattern[key['name']] = Regexp.compile(key['pattern'].gsub('?P<','?<'))
+            end
+          rescue Exception => e
+            @logger.error "Invalid subpattern regex #{e.backtrace}"
+          end
+        end
+      end 
+    end
+
+    @old_formatted_line = {}
+    @formatted_line = {}
+
+    @masking_config = if @logtype_config.has_key? 'maskingConfig' then  @logtype_config['maskingConfig']  else nil end
+    @hashing_config = if @logtype_config.has_key? 'hashingConfig' then  @logtype_config['hashingConfig']  else nil end
+    @derived_config = if @logtype_config.has_key? 'derivedConfig' then @logtype_config['derivedConfig'] else nil end
+    @general_regex = Regexp.compile("(.*)")
+    
+    if @derived_config
+      @derived_fields = {}
+      for key,value in @derived_config do
+        @derived_fields[key] = []
+        for values in @derived_config[key] do
+          @derived_fields[key].push(Regexp.compile(values.gsub('\\\\', '\\')))
+        end
+      end
+    end
+    
+    if @masking_config
+      for key,value in @masking_config do
+        @masking_config[key]["regex"] = Regexp.compile(@masking_config[key]["regex"])
+      end
+    end
+    
+    if @hashing_config
+      for key,value in @hashing_config do
+        @hashing_config[key]["regex"] = Regexp.compile(@hashing_config[key]["regex"])
+      end
+    end
+    
+    if @logtype_config.has_key?'filterConfig'
+      for field,rules in @logtype_config['filterConfig'] do
+        temp = []
+        for value in @logtype_config['filterConfig'][field]['values'] do
+          temp.push(Regexp.compile(value))
+        end
+        @logtype_config['filterConfig'][field]['values'] = temp.join('|') 
+      end
+    end 
+
     @s247_tz = {'hrs': 0, 'mins': 0} #UTC
-    @log_source = Socket.gethostname
     @valid_logtype = true
     @log_upload_allowed = true
     @log_upload_stopped_time = 0
@@ -56,20 +131,22 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
     if !@s247_datetime_format_string.include? 'unix'
       @is_year_present = if @s247_datetime_format_string.include?('%y') || @s247_datetime_format_string.include?('%Y') then true else false end
       if !@is_year_present
-	@s247_datetime_format_string = @s247_datetime_format_string+ ' %Y'
+	      @s247_datetime_format_string = @s247_datetime_format_string+ ' %Y'
       end   
       @is_timezone_present = if @s247_datetime_format_string.include? '%z' then true else false end
       if !@is_timezone_present && @logtype_config.has_key?('timezone')
-	tz_value = @logtype_config['timezone']
-	if tz_value.start_with?('+')
-	    @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('-' + tz_value[3..6])
-	elsif tz_value.start_with?('-')
-	    @s247_tz['hrs'] = Integer('+' + tz_value[1..4])
-	    @s247_tz['mins'] = Integer('+' + tz_value[3..6])
-	end
+        @s247_datetime_format_string += '%z'
+	      tz_value = @logtype_config['timezone']
+        if tz_value.start_with?('+')
+            @s247_tz['hrs'] = Integer('-' + tz_value[1..4])
+            @s247_tz['mins'] = Integer('-' + tz_value[3..6])
+        elsif tz_value.start_with?('-')
+            @s247_tz['hrs'] = Integer('+' + tz_value[1..4])
+            @s247_tz['mins'] = Integer('+' + tz_value[3..6])
+        end
       end
     end
+    Thread.new { timer_task() }
   end
 
   def init_http_client(logtype_config)
@@ -88,52 +165,96 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
         end
         datetime_string += if !@is_year_present then ' '+String(Time.new.year) else '' end
         if !@is_timezone_present && @logtype_config.has_key?('timezone')
-            @s247_datetime_format_string += '%z'
             time_zone = String(@s247_tz['hrs'])+':'+String(@s247_tz['mins'])
             datetime_string += if time_zone.start_with?('-') then time_zone else '+'+time_zone end
         end
         datetime_data = DateTime.strptime(datetime_string, @s247_datetime_format_string)
         return Integer(datetime_data.strftime('%Q'))
-    rescue
-        return 0
+    rescue Exception => e
+      @logger.error "Exception in parsing date: #{e.backtrace}"
+      return 0
+    end
+  end
+
+  def log_line_filter()
+    applyMasking()
+    applyHashing()
+    getDerivedFields()
+  end  
+
+  def get_last_group_inregex(s247_custom_regex)
+    return @s247_custom_regex.names[-1]
+  end
+
+  def remove_ignored_fields()
+    @s247_ignored_fields.each do |field_name|
+      @log_size -= if @log_fields.has_key?field_name then @log_fields.delete(field_name).bytesize else 0 end
     end
   end
 
+  def add_message_metadata()
+    @log_fields.update({'_zl_timestamp' => get_timestamp(@log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source})
+  end
+
   def parse_lines(lines)
     parsed_lines = []
-    log_size = 0
     lines.each do |line|
        if !line.empty?
-	    begin
-		if match = line.match(@s247_custom_regex)
-                    log_size += line.bytesize
-		    log_fields = match&.named_captures
-		    removed_log_size=0
-		    @s247_ignored_fields.each do |field_name|
-		        removed_log_size += if log_fields.has_key?field_name then log_fields.delete(field_name).bytesize else 0 end
-		    end
-		    formatted_line = {'_zl_timestamp' => get_timestamp(log_fields[@logtype_config['dateField']]), 's247agentuid' => @log_source}
-		    formatted_line.merge!(log_fields)
-                    parsed_lines.push(formatted_line)
-		    log_size -= removed_log_size
-                else
-                    @logger.debug("pattern not matched regex : #{@s247_custom_regex} and received line : #{line}")
-		end
-	    rescue Exception => e
-		@logger.error("Exception in parse_line #{e.backtrace}")
-	    end
-       end
+	      begin
+        @logged = false
+          match = line.match(@s247_custom_regex)
+          if match
+            @formatted_line.update(@old_formatted_line)
+            @log_size += @old_log_size
+            @old_log_size = line.bytesize
+            @log_fields = match&.named_captures
+            remove_ignored_fields()
+            add_message_metadata()
+            @old_formatted_line = @log_fields
+            @last_line_matched = true
+            @trace_started = false             
+          elsif @last_line_matched || @trace_started
+            is_date_present = !(line.scan(@datetime_regex).empty?)
+            @trace_started = !(is_date_present)
+            if !(is_date_present) && @old_formatted_line
+              if @old_formatted_line.has_key?(@message_key)
+                @old_formatted_line[@message_key] += '\n' + line
+                @old_log_size += line.bytesize
+                @trace_started = true
+                @last_line_matched = false
+              end 
+            end
+          end   
+          if @formatted_line.has_key?('_zl_timestamp')
+            log_line_filter()
+            parsed_lines.push(@formatted_line)
+            @formatted_line = {}
+          end    
+        rescue Exception => e
+          @logger.error "Exception in parse_line #{e.backtrace}"
+          @formatted_line = {}
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines  
   end
 
-  def is_filters_matched(formatted_line)
-    if @logtype_config.has_key?'filterConfig'
-        @logtype_config['filterConfig'].each do |config|
-            if formatted_line.has_key?config && (filter_config[config]['match'] ^ (filter_config[config]['values'].include?formatted_line[config]))
-                return false
-	    end
+  def is_filters_matched()
+    begin
+      if @logtype_config.has_key?'filterConfig'
+        @logtype_config['filterConfig'].each do |config,value|
+          if @formatted_line[config].scan(Regexp.new(@logtype_config['filterConfig'][config]['values'])).length > 0
+            val = true 
+          else
+            val = false
+          end
+          if (@formatted_line.has_key?config) && (@logtype_config['filterConfig'][config]['match'] ^ (val))
+            return false
+          end
         end
+      end
+    rescue Exception => e
+      @logger.error "Exception occurred in filter: #{e.backtrace}" 
     end
     return true
   end
@@ -141,7 +262,7 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
   def get_json_value(obj, key, datatype=nil)
     if obj != nil && (obj.has_key?key)
        if datatype and datatype == 'json-object'
-	  arr_json = []
+	        arr_json = []
           child_obj = obj[key]
           if child_obj.class == String
              child_obj = JSON.parse(child_obj.gsub('\\','\\\\'))
@@ -154,48 +275,151 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
          return (if obj.has_key?key then obj[key] else obj[key.downcase] end)
        end
     elsif key.include?'.'
-	parent_key = key[0..key.index('.')-1]
-	child_key = key[key.index('.')+1..-1]
-        child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
-	if child_obj.class == String
-            child_obj = JSON.parse(child_obj.replace('\\','\\\\'))
-        end
-        return get_json_value(child_obj, child_key)
+	    parent_key = key[0..key.index('.')-1]
+	    child_key = key[key.index('.')+1..-1]
+      child_obj = obj[if obj.has_key?parent_key then parent_key else parent_key.capitalize() end]
+	    if child_obj.class == String
+          child_obj = JSON.parse(child_obj.replace('\\','\\\\'))
+      end
+      return get_json_value(child_obj, child_key,datatype)
     end
   end
 
+  def json_log_applier(line)
+    json_log_size=0
+    @formatted_line = {}
+    @log_fields = {}
+    event_obj = if line.is_a?(String) then JSON.parse(line) else line end
+    @logtype_config['jsonPath'].each do |path_obj|
+      value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
+      if value
+        @log_fields[path_obj['name']] = value 
+        json_log_size+= String(value).bytesize - (if value.class == Array then value.size*20 else 0 end)
+      end
+    end
+    for key,regex in @sub_pattern do
+      if @log_fields.has_key?(key)
+        matcher = regex.match(@log_fields.delete(key))
+        if matcher
+          @log_fields.update(matcher.named_captures)
+          remove_ignored_fields()
+          @formatted_line.update(@log_fields)
+        end 
+      end
+    end
+
+    if !(is_filters_matched())
+      return false
+    else
+      add_message_metadata()
+      @formatted_line.update(@log_fields)
+      log_line_filter()
+      @log_size += json_log_size
+      return true
+    end
+  end
+  
   def json_log_parser(lines_read)
-    log_size = 0
     parsed_lines = []
     lines_read.each do |line|
+      begin
+        @logged = false
         if !line.empty?
-          current_log_size = 0
-	  formatted_line = {}
-	  event_obj = Yajl::Parser.parse(line)
-	  @logtype_config['jsonPath'].each do |path_obj|
-	    value = get_json_value(event_obj, path_obj[if path_obj.has_key?'key' then 'key' else 'name' end], path_obj['type'])
-            if value
-	      formatted_line[path_obj['name']] = value 
-	      current_log_size+= String(value).size - (if value.class == Array then value.size*20 else 0 end)
+          if ((line[0] == '{') && (@json_data[-1] == '}'))
+            if json_log_applier(@json_data)
+              parsed_lines.push(@formatted_line)
             end
+            @json_data=''
           end
-	  if is_filters_matched(formatted_line)
-	    formatted_line['_zl_timestamp'] = get_timestamp(formatted_line[@logtype_config['dateField']])
-	    formatted_line['s247agentuid'] = @log_source
-	    parsed_lines.push(formatted_line)
-            log_size+=current_log_size
+          @json_data += line
+        end
+      rescue Exception => e
+        @logger.error "Exception in parse_line #{e.backtrace}"
+      end
+    end
+    return parsed_lines
+  end
+
+  def ml_regex_applier(ml_trace, ml_data)
+    begin    
+      @log_size += @ml_trace.bytesize
+      matcher = @s247_custom_regex.match(@ml_trace)  
+      @log_fields = matcher.named_captures
+      @log_fields.update(@ml_data)
+      if @s247_ignored_fields
+        remove_ignored_fields()
+      end
+      add_message_metadata()
+      @formatted_line.update(@log_fields)
+      log_line_filter()
+    rescue Exception => e
+      @logger.error "Exception occurred in ml_parser : #{e.backtrace}" 
+      @formatted_line = {}
+    end
+  end
+
+  def ml_log_parser(lines)
+    parsed_lines = []
+    lines.each do |line|
+      if !line.empty?
+        begin
+          @logged = false
+          ml_start_matcher = @ml_regex.match(line)
+          if ml_start_matcher || @ml_end_line_found
+            @ml_found = ml_start_matcher
+            @ml_end_line_found = false
+            @formatted_line = {}
+            if @ml_trace.length > 0 
+              begin
+                  ml_regex_applier(@ml_trace, @ml_data)
+                  if @ml_trace_buffer && @formatted_line
+                    @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
+                    @log_size += @ml_trace_buffer.bytesize
+                  end
+                  parsed_lines.push(@formatted_line)
+                  @ml_trace = ''
+                  @ml_trace_buffer = ''
+                  if @ml_found
+                    @ml_data = ml_start_matcher.named_captures
+                    @log_size += line.bytesize
+                  else
+                      @ml_data = {}
+                  end
+                  @ml_count = 0
+                rescue Exception => e
+                  @logger.error "Exception occurred in ml_parser : #{e.backtrace}"
+                end
+              elsif @ml_found
+                @log_size += line.bytesize
+                @ml_data = ml_start_matcher.named_captures
+              end
+          elsif @ml_found
+            if @ml_count < @max_ml_count
+              @ml_trace += '<NewLine>' + line
+            elsif @ml_end_regex && @ml_end_regex.match(line)
+              @ml_end_line_found = True
+            elsif (@ml_count - @max_ml_count) < @max_trace_line
+              @ml_trace_buffer += "\n" + line
+            end
+            @ml_count += 1
           end
-	end
+        rescue Exception => e
+          @logger.error "Exception occurred in ml_parser : #{e.backtrace}"
+        end
+      end
     end
-    return parsed_lines, log_size
+    return parsed_lines
   end
-  
+
   def process_http_events(events)
+    @before_time = Time.now
     batches = batch_http_events(events)
     batches.each do |batched_event|
-      formatted_events, log_size = format_http_event_batch(batched_event)
-      formatted_events = gzip_compress(formatted_events)
-      send_logs_to_s247(formatted_events, log_size)
+      formatted_events, @log_size = format_http_event_batch(batched_event)
+      if (formatted_events.length>0)
+        formatted_events = gzip_compress(formatted_events)
+        send_logs_to_s247(formatted_events, @log_size)
+      end
     end
   end
 
@@ -204,12 +428,12 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
     current_batch = []
     current_batch_size = 0
     encoded_events.each_with_index do |encoded_event, i|
-      event_message = encoded_event.to_hash['message']
+      event_message = if encoded_event.to_hash().has_key? 'AL_PARSED' then encoded_event.to_hash() else encoded_event.to_hash['message'] end
+    current_event_size = if event_message.is_a?(Hash) then event_message.to_s.bytesize else event_message.bytesize end
+    if current_event_size > S247_MAX_RECORD_SIZE
+      event_message = event_message[0..(S247_MAX_RECORD_SIZE-DD_TRUNCATION_SUFFIX.length)]+DD_TRUNCATION_SUFFIX
       current_event_size = event_message.bytesize
-      if current_event_size > S247_MAX_RECORD_SIZE
-        event_message = event_message[0..(S247_MAX_RECORD_SIZE-DD_TRUNCATION_SUFFIX.length)]+DD_TRUNCATION_SUFFIX
-        current_event_size = event_message.bytesize
-      end
+    end
 
       if (i > 0 and i % S247_MAX_RECORD_COUNT == 0) or (current_batch_size + current_event_size > S247_MAX_BATCH_SIZE)
         batches << current_batch
@@ -218,7 +442,7 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
       end
 
       current_batch_size += current_event_size
-      current_batch << event_message
+      current_batch <<  (if event_message.is_a?(Hash) then event_message.to_json.to_s else event_message end)    
     end
     batches << current_batch
     batches
@@ -226,13 +450,19 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
 
   def format_http_event_batch(events)
     parsed_lines = []
-    log_size = 0
+    @log_size = 0
+    @old_log_size=0    
     if @logtype_config.has_key?'jsonPath'
-      parsed_lines, log_size = json_log_parser(events)
+      parsed_lines = json_log_parser(events)
+    elsif @logtype_config.has_key?'ml_regex'
+      parsed_lines = ml_log_parser(events)
     else
-       parsed_lines, log_size = parse_lines(events)
+      parsed_lines = parse_lines(events)
+    end
+    if (parsed_lines.length > 0)
+      return JSON.dump(parsed_lines), @log_size
     end
-    return LogStash::Json.dump(parsed_lines), log_size
+    return [],0  
   end
 
   def gzip_compress(payload)
@@ -248,7 +478,7 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
   end
 
   def send_logs_to_s247(gzipped_parsed_lines, log_size)
-     @headers['Log-Size'] = String(log_size)
+     @headers['Log-Size'] = String(@log_size)
      sleep_interval = @retry_interval
      begin
         @max_retry.times do |counter|
@@ -260,17 +490,17 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
               if resp_headers.has_key?'LOG_LICENSE_EXCEEDS' && resp_headers['LOG_LICENSE_EXCEEDS'] == 'True'
                 @logger.error("Log license limit exceeds so not able to send logs")
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'BLOCKED_LOGTYPE' && resp_headers['BLOCKED_LOGTYPE'] == 'True'
                 @logger.error("Max upload limit reached for log type")
                 @log_upload_allowed = false
-		@log_upload_stopped_time =Time.now.to_i
+		            @log_upload_stopped_time =Time.now.to_i
               elsif resp_headers.has_key?'INVALID_LOGTYPE' && resp_headers['INVALID_LOGTYPE'] == 'True'
                 @logger.error("Log type not present in this account so stopping log collection")
-		@valid_logtype = false
+		            @valid_logtype = false
               else
-		@log_upload_allowed = true
-                @logger.debug("Successfully sent logs with size #{gzipped_parsed_lines.size} / #{log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}")
+		            @log_upload_allowed = true
+                @logger.debug("Successfully sent logs with size #{gzipped_parsed_lines.size} / #{@log_size} to site24x7. Upload Id : #{resp_headers['x-uploadid']}")
               end
             else
               @logger.error("Response Code #{response.code} from Site24x7, so retrying (#{counter + 1}/#{@max_retry})")
@@ -296,4 +526,161 @@ class LogStash::Outputs::Site24x7 < LogStash::Outputs::Base
       end
   end
 
-end
+  def log_the_holded_line()
+    @log_size = 0
+    if @logged == false
+      if (@ml_trace.length>0)
+        ml_regex_applier(@ml_trace, @ml_data)
+        if @ml_trace_buffer
+          if !(@formatted_line.empty?)
+            @formatted_line[@message_key] = @formatted_line[@message_key] + @ml_trace_buffer
+            @log_size += @ml_trace_buffer.bytesize  
+          else
+            @ml_trace += @ml_trace_buffer.gsub('\n', '<NewLine>')
+            ml_regex_applier(@ml_trace, @ml_data)
+          end
+          @ml_trace_buffer = ''
+        end    
+        @ml_trace = ''
+      elsif (@json_data.length>0)
+        if !(json_log_applier(@json_data))
+          @formatted_line={}
+        end
+        @json_data = ''
+      elsif @old_formatted_line
+        @formatted_line.update(@old_formatted_line)
+        log_line_filter()
+        @log_size += @old_log_size
+        @old_formatted_line = {}
+        @old_log_size = 0
+      end
+      @logged = true
+      if @format_record
+        @custom_parser.format_record()
+      end
+      if !(@formatted_line.empty?)  
+        return @formatted_line
+      end
+    end
+      return nil
+  end
+
+  def applyMasking()
+    if @masking_config
+      begin
+        for key,value in @masking_config do
+          adjust_length = 0
+          mask_regex = @masking_config[key]["regex"]
+          if @formatted_line.has_key?key
+            field_value = @formatted_line[key]
+            if !(mask_regex.eql?(@general_regex))
+              matcher = field_value.to_enum(:scan, mask_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0]  
+                  _end = matcher[index].offset(1)[1]
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    adjust_length += (_end - start) - @masking_config[key]['string'].bytesize
+                    field_value = field_value[0..(start-1)] + @masking_config[key]['string'] + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+              @formatted_line[key] = field_value
+              @log_size -= adjust_length
+            else
+              @log_size -= (@formatted_line[key].bytesize - @masking_config[key]['string'].bytesize)
+              @formatted_line[key] = @masking_config[key]['string']
+            end
+          end
+        end
+      rescue Exception => e
+        @logger.error "Exception occurred in masking : #{e.backtrace}"
+      end      
+    end  
+  end
+  
+  def applyHashing()
+    if @hashing_config
+      begin
+        for key,value in @hashing_config do
+          hash_regex = @hashing_config[key]["regex"]
+          if @formatted_line.has_key?key
+            field_value = @formatted_line[key]
+            if (hash_regex.eql?(@general_regex))
+              hash_string =  Digest::SHA256.hexdigest(field_value)
+              field_value = hash_string
+            else  
+              adjust_length = 0
+              matcher = field_value.to_enum(:scan, hash_regex).map { Regexp.last_match }
+              if matcher
+                (0..(matcher.length)-1).map do |index| 
+                  start = matcher[index].offset(1)[0] 
+                  _end = matcher[index].offset(1)[1] 
+                  if ((start >= 0) && (_end > 0))
+                    start = start - adjust_length
+                    _end = _end - adjust_length
+                    hash_string =  Digest::SHA256.hexdigest(field_value[start..(_end-1)])
+                    adjust_length += (_end - start) - hash_string.bytesize
+                    field_value = field_value[0..(start-1)] + hash_string + field_value[_end..field_value.bytesize]
+                  end
+                end
+              end
+            end
+            if adjust_length
+              @log_size -= adjust_length
+            else
+              @log_size -= (@formatted_line[key].bytesize - field_value.bytesize)
+            end
+            @formatted_line[key] = field_value
+          end
+        end
+      rescue Exception => e
+        @logger.error "Exception occurred in hashing : #{e.backtrace}"
+      end    
+    end
+  end
+  
+  def getDerivedFields()
+    if @derived_config
+      begin
+        for key,value in @derived_fields do
+          for each in @derived_fields[key] do
+              if @formatted_line.has_key?key
+                match_derived = each.match(@formatted_line[key])
+                if match_derived
+                  @formatted_line.update(match_derived.named_captures)
+                  for field_name,value in match_derived.named_captures do
+                    @log_size += @formatted_line[field_name].bytesize
+                  end  
+                end
+                break
+              end
+          end
+        end
+      rescue Exception => e
+        @logger.error "Exception occurred in derived fields : #{e.backtrace}"
+      end    
+    end
+  end
+  
+  def timer_task()
+    while true
+      @after_time = Time.now
+      if @before_time
+        diff = @after_time-@before_time
+        if diff.to_i > 29
+            out = log_the_holded_line()
+            if out != nil
+              out = JSON.dump([out])
+              out = gzip_compress(out)
+              send_logs_to_s247(out, @log_size)
+            end
+        end
+      end
+      sleep(30)
+    end
+  end
+
+end

data/logstash-output-site24x7.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-site24x7'
-  s.version       = '0.1.0'
+  s.version       = '0.1.2'
   s.licenses      = ['']
   s.summary       = 'Site24x7 output plugin for Logstash event collector'
   s.homepage      = 'https://github.com/site24x7/logstash-output-site24x7'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-site24x7
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Magesh Rajan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-11 00:00:00.000000000 Z
+date: 2022-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Site24x7 output plugin for Logstash event collector