logstash-input-azure_blob_storage 0.12.5 → 0.12.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00e66bdb4eda73c6d9a4219034d6a33bbf4d8a1c8206f7f2e6ec39b414dd9d63
-  data.tar.gz: 953fa1cc28b60e5a44d7575ead5428e6fd62c309ad1c9630a2f2cd73dac1ffc3
+  metadata.gz: b50189c380606c6fdb8b7f7216fe20d15c0d410f1c1f6670211baf25baa567ca
+  data.tar.gz: 189c80c15720ec9a85b8bb223a5ae7e4666fd0ebd6a96946f201bee96cf3dafc
 SHA512:
-  metadata.gz: a19ff34ae098f9bf115789b43781c4073268934c34fd69a2ae119ea844deffcd30d853aefc29e00fe4495858cbf336ec1c6dc0f2113c26239a47fcacfb73bb87
-  data.tar.gz: 93ff0a91bfc54f8b159c80c9e0156064b3805166c0d83ae51fae09a8944e6f6b852ed72d269d5b1dd6e8561007a80042583fe2f7d7be20b09122c414c13ff94b
+  metadata.gz: 599ca22fd813634d3ffd5fbbef0361605fd7611ea4050bc85e30c06fe97dbfe6dcd879ee092573e8a94229435d25c7cef71255bc72f33ea3d4813de987600e4c
+  data.tar.gz: 53cc0e73c25323ba891e90a820c679071516187d641ed2c5dd5810a5bbb9654c2cf67c6239d400b58d8786c4cc4737aaa54b0fc1f145b4136ebf1f6b0203a00d

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## PROBABLY 0.12.4 is the most stable version until I sort out when and why JSON Parse errors happen
+Join the discussion if you have something to share!
+https://github.com/janmg/logstash-input-azure_blob_storage/issues/34
+
+
+## 0.12.6
+  - Fixed the 0.12.5 exception handling, it actually caused a warning to become a fatal pipeline crashing error
+  - The chuck that failed to process should be printed in debug mode, for testing use debug_until => 10000 
+  - Now check if registry entry exist before loading the offsets, to avoid caught: undefined method `[]' for nil:NilClass
+
+## 0.12.5
+  - Added exception message on json parse errors
+
 ## 0.12.4
   - Connection Cache reset removed, since agents are cached per host
   - Explicit handling of json_lines and respecting line boundaries (thanks nttoshev)

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -205,10 +205,12 @@ public
             filelist = list_blobs(false)
             filelist.each do |name, file|
                 off = 0
-                begin
+                if @registry.key?(name) then
+                  begin
                     off = @registry[name][:offset]
-                rescue Exception => e
+                  rescue Exception => e
                     @logger.error("caught: #{e.message} while reading #{name}")
+                  end
                 end
                 @registry.store(name, { :offset => off, :length => file[:length] })
                 if (@debug_until > @processed) then @logger.info("2: adding offsets: #{name} #{off} #{file[:length]}") end
@@ -271,9 +273,9 @@ public
                                 fingjson = JSON.parse(chunk)
                                 @processed += nsgflowlog(queue, fingjson, name)
                                 @logger.debug("Processed #{res[:nsg]} [#{res[:date]}] #{@processed} events")
-                            rescue JSON::ParserError
+                            rescue JSON::ParserError => e
                                 @logger.error("parse error #{e.message} on #{res[:nsg]} [#{res[:date]}] offset: #{file[:offset]} length: #{file[:length]}")
-                                @logger.debug("#{chunk}")
+                                if (@debug_until > @processed) then @logger.info("#{chunk}") end
                             end
                         end
                     # TODO: Convert this to line based grokking.
@@ -573,7 +575,7 @@ private
                             @logger.debug("learned tail: #{@tail}")
                         end
                     rescue Exception => e
-                        @logger.info("learn json one of the attempts failed #{e.message}")
+                        @logger.info("learn json one of the attempts failed")
                     end
                 end
             end

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
     s.name          = 'logstash-input-azure_blob_storage'
-    s.version       = '0.12.5'
+    s.version       = '0.12.6'
     s.licenses      = ['Apache-2.0']
     s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
     s.description   = <<-EOF
@@ -23,7 +23,6 @@ EOF
     s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
     s.add_runtime_dependency 'stud', '~> 0.0.23'
     s.add_runtime_dependency 'azure-storage-blob', '~> 2', '>= 2.0.3'
-
-    s.add_development_dependency 'logstash-devutils'
-    s.add_development_dependency 'rubocop'
+    s.add_development_dependency 'logstash-devutils', '~> 2.4'
+    s.add_development_dependency 'rubocop', '~> 1.48'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.12.5
+  version: 0.12.6
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-08 00:00:00.000000000 Z
+date: 2023-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,31 +61,31 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.48'
   name: rubocop
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.48'
 description: " This gem is a Logstash plugin. It reads and parses data from Azure\
   \ Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob\
   \ from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial\