logstash-output-application_insights 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 771238e72676111249e1ff68b5d04dd9c3fdad43
-  data.tar.gz: d5d25c0a1e678c27e9b034f1a63f5925cdd0ddfc
+  metadata.gz: 58f3bc06a6e48003f3349c1389268c9101a96775
+  data.tar.gz: c068a8bdb28e647e7b456b899128fbf379183de9
 SHA512:
-  metadata.gz: d66c4886bb6aa1b08f0840b4090cb5d8874ca12af83de60b3bad38dfe7fbcc668215f2bc57b84ecc8ba075e26bbdcd94fefd6fa807990378823683432fe5135c
-  data.tar.gz: bfd12398aaf0c5a82d0256b71b9207af816bac5715ea4812079c04ebce46a73b392f0befae7b5c51d0215fbe1a779c6cf625e460e33be131d8d3a2dbd101d324
+  metadata.gz: b57f47e54e2a3d32fbf3a3548f85b8cfb90e8f3bda876d25b186c515ab3407fd3f47eba03c44487bf260506195fa8eb217912b6a80e445042112f223460da472
+  data.tar.gz: 7c93888e412e7515c2085259fbe1687a5e334eaad97404bd390f17431cef9db0a89c09abce6f3a79f6a4a53347b54c63152246598b62805c6c164c8e5f3d0a4e

data/README.md

@@ -414,6 +414,14 @@ example:
 disable_blob_upload => true
 ```
 
+### disable_truncation
+When set to true, event fields won't be truncated to max 1MB (beware: The max allows bytes size per filed is 1MB, setting it to true, it will be just waste of bandwidth and storage) Default false
+Used for troubleshooting
+example:
+```ruby
+disable_truncation => true
+```
+
 ### stop_on_unknown_io_errors
 When set to true, process will stop if an unknown IO error is detected. Default false
 Used for troubleshooting

data/lib/logstash/outputs/application_insights.rb

@@ -305,6 +305,9 @@ class LogStash::Outputs::Application_insights < LogStash::Outputs::Base
 
   # When set to true, blobs won't be compressed. 
   config :disable_compression, :validate => :boolean
+
+  # When set to true, blobs won't be compressed. 
+  config :disable_truncation, :validate => :boolean
   public
 
   def register

data/lib/logstash/outputs/application_insights/channel.rb

@@ -34,6 +34,7 @@ class LogStash::Outputs::Application_insights
       @closing = false
       configuration = Config.current
       
+      @disable_truncation = configuration[:disable_truncation]
       @file_pipe = !configuration[:disable_compression]
       @gzip_file = !configuration[:disable_compression]
       @blob_max_bytesize = configuration[:blob_max_bytesize]
@@ -100,7 +101,7 @@ class LogStash::Outputs::Application_insights
         sub_channel = @workers_channel[Thread.current] || @semaphore.synchronize { @workers_channel[Thread.current] = Sub_channel.new( @event_separator ) }
         sub_channel << serialized_event
       else
-        @logger.warn { "event not uploaded, no relevant data in event. table_id: #{table_id}, event: #{data}" }
+        @logger.warn { "event not uploaded, no relevant data in event. table_id: #{@table_id}, event: #{data}" }
       end
     end
 
@@ -265,7 +266,7 @@ class LogStash::Outputs::Application_insights
         serialized_data = data
       elsif EXT_EVENT_FORMAT_CSV == @event_format
         if data.is_a?( Array )
-          serialized_data = data.to_csv( :col_sep => @csv_separator )
+          serialized_data = serialize_array_to_csv( data )
         elsif data.is_a?( Hash )
           serialized_data = serialize_to_csv( data )
         end
@@ -281,16 +282,26 @@ class LogStash::Outputs::Application_insights
 
 
     def serialize_to_json ( data )
-      return data.to_json unless !@table_columns.nil?
-
-      data = Utils.downcase_hash_keys( data ) if @case_insensitive_columns
+      if (@table_columns.nil?)
+        json_hash = data
+      else
+        data = Utils.downcase_hash_keys( data ) if @case_insensitive_columns
 
-      json_hash = {  }
-      @table_columns.each do |column|
-        value = data[column[:field_name]] || column[:default]
-        json_hash[column[:name]] = value if value
+        json_hash = {  }
+        @table_columns.each do |column|
+          value = data[column[:field_name]] || column[:default]
+          json_hash[column[:name]] = truncate_if_too_big( value ) if value
+        end
       end
+
       return nil if json_hash.empty?
+
+      json_string = json_hash.to_json
+      return json_string if json_string.bytesize < MAX_FIELD_BYTES || @disable_truncation
+
+      json_hash.each_pair do |name, value|
+        json_hash[name] = truncate_data_if_too_big( name, value )
+      end
       json_hash.to_json
     end
 
@@ -306,11 +317,80 @@ class LogStash::Outputs::Application_insights
         type = (column[:type] || value.class.name).downcase.to_sym
         csv_array << ( [:hash, :array, :json, :dynamic, :object].include?( type ) ? value.to_json : value )
       end
+      serialize_array_to_csv( csv_array )
+    end
+
+    def serialize_array_to_csv ( csv_array )
       return nil if csv_array.empty?
+      csv_string = csv_array.to_csv( :col_sep => @csv_separator )
+      return csv_string if csv_string.bytesize < MAX_FIELD_BYTES || @disable_truncation
+
+      index = 0
+      csv_array.map! do |value|
+        index += 1
+        truncate_data_if_too_big( index.to_s, value )
+      end
       csv_array.to_csv( :col_sep => @csv_separator )
     end
 
 
+    def truncate_data_if_too_big ( name, data )
+      return data if @disable_truncation
+
+      truncated = nil
+      if data.is_a?( String )
+        if data.bytesize > MAX_FIELD_BYTES
+          truncated = data.bytesize - MAX_FIELD_BYTES
+          data = data.byteslice( 0, MAX_FIELD_BYTES )
+        end
+      elsif data.is_a?( Hash )
+        str = data.to_json
+        while str.bytesize > MAX_FIELD_BYTES
+          truncated = str.bytesize - MAX_FIELD_BYTES unless truncated
+          delta = str.bytesize - MAX_FIELD_BYTES
+          max_size = 0
+          max_name = nil
+          data.each_pair do |name, value|
+            if value.is_a?( String ) && value.bytesize > max_size
+              max_name = name
+              max_size = value.bytesize
+            end
+          end
+          unless max_name
+            data = {}
+            break
+          end
+          data[max_name] = data[max_name].byteslice( 0,  max_size - ( max_size > delta ? delta : max_size ) )
+          str = data.to_json
+        end
+
+      elsif data.is_a?( Array )
+        str = data.to_json
+        while str.bytesize > MAX_FIELD_BYTES
+          truncated = str.bytesize - MAX_FIELD_BYTES unless truncated
+          delta = str.bytesize - MAX_FIELD_BYTES
+          max_size = 0
+          max_index = nil
+          data.each_index do |index|
+            value = data[index]
+            if value.is_a?( String ) && value.bytesize > max_size
+              max_index = index
+              max_size = value.bytesize
+            end
+          end
+          unless max_index
+            data = []
+            break
+          end
+          data[max_index] = data[max_index].byteslice( 0,  max_size - ( max_size > delta ? delta : max_size ) )
+          str = data.to_json
+        end
+      end
+
+      @logger.warn { "field #{name} was truncated by #{truncated} bytes, due to size above #{MAX_FIELD_BYTES} bytes. table_id: #{@table_id}" } if truncated
+      data
+    end
+
     def find_upload_pipe
       min_upload_pipe = @active_upload_pipes[0]
       @active_upload_pipes.each do |upload_pipe|

data/lib/logstash/outputs/application_insights/constants.rb

@@ -74,6 +74,7 @@ class LogStash::Outputs::Application_insights
       :csv_separator => @csv_separator || DEFAULT_CSV_SEPARATOR,
       :csv_default_value => @csv_default_value || DEFAULT_CSV_DEFAULT_VALUE,
       :disable_compression => @disable_compression || DEFAULT_DISABLE_COMPRESSION,
+      :disable_truncation => @disable_truncation || DEFAULT_DISABLE_TRUNCATION,
 
     }
   end
@@ -83,7 +84,7 @@ class LogStash::Outputs::Application_insights
                           :disable_cleanup, :delete_not_notified_blobs,
                           :validate_notification, :validate_storage,
                           :save_notified_blobs_records, :case_insensitive_columns,
-                          :disable_compression,
+                          :disable_compression, :disable_truncation,
                           :table_columns, :serialized_event_field ]
 
   GUID_NULL =                    "00000000-0000-0000-0000-000000000000"
@@ -144,6 +145,7 @@ class LogStash::Outputs::Application_insights
 
   MAX_CHANNEL_UPLOAD_PIPES = 40
   CHANNEL_THRESHOLD_TO_ADD_UPLOAD_PIPE = 3                # not relevant for file upload mode
+  MAX_FIELD_BYTES = 1024 * 1024
 
   METADATA_FIELD_INSTRUMENTATION_KEY = "[@metadata]instrumentation_key"
   METADATA_FIELD_TABLE_ID = "[@metadata]table_id"
@@ -202,6 +204,7 @@ class LogStash::Outputs::Application_insights
   DEFAULT_DELETE_NOT_NOTIFIED_BLOBS = false
   DEFAULT_SAVE_NOTIFIED_BLOBS_RECORDS = false
   DEFAULT_DISABLE_COMPRESSION = false
+  DEFAULT_DISABLE_TRUNCATION = false
   
   DEFAULT_CASE_INSENSITIVE = false
 

data/lib/logstash/outputs/application_insights/version.rb

@@ -20,5 +20,5 @@
 # ----------------------------------------------------------------------------------
 
 # class LogStash::Outputs::Application_insights
-  APPLICATION_INSIGHTS_VERSION ||= "0.2.2"
+  APPLICATION_INSIGHTS_VERSION ||= "0.2.3"
 # end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-application_insights
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-20 00:00:00.000000000 Z
+date: 2016-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement