logstash-input-azure_blob_storage 0.11.6 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ececd96b04d2cab60eca54a0fe2a98c9ed093da2227e3568d4feea09264912fa
-  data.tar.gz: 7bcd39bc38d26a05da1275e5fb2317e41b5c2cddc6541535c7d166a69bb3cf62
+  metadata.gz: 7d9a45fa4fa24b7e14ffd518665c9310bbf4ff6ba82fd063fde124a45d20f475
+  data.tar.gz: 56b119316882fefc3d208418aac0b1b712d94649099297f52bef1f04eb44f7fb
 SHA512:
-  metadata.gz: 1bcbfab30de973e9eafee295221dc816411dca0e0f747a01c62bb48ec5c46eaf4db4162fdd5283611cd79da59910daab9e7c6e234df47f5ce7f320e65f7b8c69
-  data.tar.gz: 7bbdab8694d024b9c08cc89e13bc86aa8b90a536f5615565333593e0da7c3073d7c4cf3ad3f2b4005a90541de9693a93826158a18fbf9015234bee1812b3d46c
+  metadata.gz: 17f9c1fc032e18a764a6985a4e6d5a96eacfb4732861a9f6c1a6f41be01ff9c9036e0ce5d15348da12a3d236855111744e6bd867e64162a7417e4f8bdd862392
+  data.tar.gz: 6e39fc13efa7f7aea54e6aad9c4d5e1c3c00e1c9a5c56f39b28622496b598aecada794c337f9c348c911527b1e17229d1923829a707acf4bfa282da94441dd46

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+#$ 0.12.2
+  - Fixed the exception handling, not trying to print how many JSON fields there are while catching the exception
+
+## 0.12.1
+  - Catch NSGFLOW logs when the JSON parsing somehow failed
+
+## 0.12.0
+  - version 2 of azure-storage
+  - saving current files registry, not keeping historical files
+
+## 0.11.7
+  - implemented skip_learning
+  - start ignoring failed files and not retry
+
 ## 0.11.6
   - fix in json head and tail learning the max_results
   - broke out connection setup in order to call it again if connection exceptions come

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -213,7 +213,8 @@ def run(queue)
 	end
         # size nilClass when the list doesn't grow?!
         # Worklist is the subset of files where the already read offset is smaller than the file size
-	worklist.clear
+        @registry = newreg
+        worklist.clear
         chunk = nil
 
 	worklist = newreg.select {|name,file| file[:offset] < file[:length]}
@@ -235,7 +236,9 @@ def run(queue)
                         chunk = full_read(name)
                         size=chunk.size
                     rescue Exception => e
-                        @logger.error("Failed to read #{name} because of: #{e.message} .. will continue and pretend this never happened")
+                        @logger.error("Failed to read #{name} because of: #{e.message} .. will continue, set file as read and pretend this never happened")
+                        @logger.error("#{size} size and #{file[:length]} file length")
+                        size = file[:length]
                     end 
                 else
                     @logger.info("found a zero size file #{name}")
@@ -386,6 +389,7 @@ end
 
 def nsgflowlog(queue, json, name)
     count=0
+    begin
     json["records"].each do |record|
       res = resource(record["resourceId"])
       resource = { :subscription => res[:subscription], :resourcegroup => res[:resourcegroup], :nsg => res[:nsg] }
@@ -415,6 +419,9 @@ def nsgflowlog(queue, json, name)
           end
       end
     end
+    rescue Exception => e
+      @logger.error("NSG Flowlog problem for #{name} and error message #{e.message}")
+    end
     return count 
 end
 

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.11.6'
+  s.version       = '0.12.2'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF
@@ -22,6 +22,6 @@ EOF
   # Gem dependencies
   s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.1'
   s.add_runtime_dependency 'stud', '~> 0.0.23'
-  s.add_runtime_dependency 'azure-storage-blob', '~> 1.1'
+  s.add_runtime_dependency 'azure-storage-blob', '~> 2', '>= 2.0.3'
   #s.add_development_dependency 'logstash-devutils', '~> 2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.11.6
+  version: 0.12.2
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,8 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.1'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -31,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.23
   name: stud
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -43,15 +43,21 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
   name: azure-storage-blob
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
 description: " This gem is a Logstash plugin. It reads and parses data from Azure\
   \ Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob\
   \ from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial\
@@ -92,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.