logstash-input-azure_blob_storage 0.12.4 → 0.12.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec2c4d4ee6fcbbec7e34682089aeaa4c4b212dbb78bc9f6eab63fb4f08bbfd6a
-  data.tar.gz: f041887728e35433066791d78a2eae76bf564bcf5e87f4cc199720f1cd1910e4
+  metadata.gz: b50189c380606c6fdb8b7f7216fe20d15c0d410f1c1f6670211baf25baa567ca
+  data.tar.gz: 189c80c15720ec9a85b8bb223a5ae7e4666fd0ebd6a96946f201bee96cf3dafc
 SHA512:
-  metadata.gz: 4d745d3fa9ed1c768d2ca8c59b8d9034b4c56294b56e19796c9f622e4fb695c8050ab63c0e961fb1bd66c41b67999a37c42e2f0c49145dc5bf9d6c267a00d334
-  data.tar.gz: d7d9a1dbd9f13c44bfeeb600abe82d2e4019e0ca5e5f55974c41777621fc56aedfc2a32d5f272d711e79a711c5e799d84cd5c9056395fee2dccb8b9c0df8bb2a
+  metadata.gz: 599ca22fd813634d3ffd5fbbef0361605fd7611ea4050bc85e30c06fe97dbfe6dcd879ee092573e8a94229435d25c7cef71255bc72f33ea3d4813de987600e4c
+  data.tar.gz: 53cc0e73c25323ba891e90a820c679071516187d641ed2c5dd5810a5bbb9654c2cf67c6239d400b58d8786c4cc4737aaa54b0fc1f145b4136ebf1f6b0203a00d

data/CHANGELOG.md

@@ -1,4 +1,18 @@
+## PROBABLY 0.12.4 is the most stable version until I sort out when and why JSON Parse errors happen
+Join the discussion if you have something to share!
+https://github.com/janmg/logstash-input-azure_blob_storage/issues/34
+
+
+## 0.12.6
+  - Fixed the 0.12.5 exception handling, it actually caused a warning to become a fatal pipeline crashing error
+  - The chuck that failed to process should be printed in debug mode, for testing use debug_until => 10000 
+  - Now check if registry entry exist before loading the offsets, to avoid caught: undefined method `[]' for nil:NilClass
+
+## 0.12.5
+  - Added exception message on json parse errors
+
 ## 0.12.4
+  - Connection Cache reset removed, since agents are cached per host
   - Explicit handling of json_lines and respecting line boundaries (thanks nttoshev)
   - Removed reprocessing on any exception
 

data/README.md

@@ -207,7 +207,7 @@ filter {
     remove_field => ["subresponse"]
     remove_field => ["username"]
     remove_field => ["clientPort"]
-    remove_field => ["port"]:0
+    remove_field => ["port"]
     remove_field => ["timestamp"]
   }
 }

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -205,10 +205,12 @@ public
             filelist = list_blobs(false)
             filelist.each do |name, file|
                 off = 0
-                begin
+                if @registry.key?(name) then
+                  begin
                     off = @registry[name][:offset]
-                rescue Exception => e
+                  rescue Exception => e
                     @logger.error("caught: #{e.message} while reading #{name}")
+                  end
                 end
                 @registry.store(name, { :offset => off, :length => file[:length] })
                 if (@debug_until > @processed) then @logger.info("2: adding offsets: #{name} #{off} #{file[:length]}") end
@@ -271,8 +273,9 @@ public
                                 fingjson = JSON.parse(chunk)
                                 @processed += nsgflowlog(queue, fingjson, name)
                                 @logger.debug("Processed #{res[:nsg]} [#{res[:date]}] #{@processed} events")
-                            rescue JSON::ParserError
-                                @logger.error("parse error on #{res[:nsg]} [#{res[:date]}] offset: #{file[:offset]} length: #{file[:length]}")
+                            rescue JSON::ParserError => e
+                                @logger.error("parse error #{e.message} on #{res[:nsg]} [#{res[:date]}] offset: #{file[:offset]} length: #{file[:length]}")
+                                if (@debug_until > @processed) then @logger.info("#{chunk}") end
                             end
                         end
                     # TODO: Convert this to line based grokking.
@@ -572,7 +575,7 @@ private
                             @logger.debug("learned tail: #{@tail}")
                         end
                     rescue Exception => e
-                        @logger.info("learn json one of the attempts failed #{e.message}")
+                        @logger.info("learn json one of the attempts failed")
                     end
                 end
             end

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name          = 'logstash-input-azure_blob_storage'
-    s.version       = '0.12.4'
+    s.version       = '0.12.6'
     s.licenses      = ['Apache-2.0']
     s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
     s.description   = <<-EOF
@@ -23,7 +23,6 @@ EOF
     s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
     s.add_runtime_dependency 'stud', '~> 0.0.23'
     s.add_runtime_dependency 'azure-storage-blob', '~> 2', '>= 2.0.3'
-
-    s.add_development_dependency 'logstash-devutils'
-    s.add_development_dependency 'rubocop'
+    s.add_development_dependency 'logstash-devutils', '~> 2.4'
+    s.add_development_dependency 'rubocop', '~> 1.48'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.12.6
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-20 00:00:00.000000000 Z
+date: 2023-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,31 +61,31 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.48'
   name: rubocop
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.48'
 description: " This gem is a Logstash plugin. It reads and parses data from Azure\
   \ Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob\
   \ from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial\
@@ -126,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.