logstash-input-azure_blob_storage 0.11.2 → 0.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b721a6aa74f4e9df285f62f47efa42112e540d9836391b31e74daf6544e1087d
-  data.tar.gz: 5d22a077d53698807a51dde75ac6c7deb273f0fe68d7ea05a46651b5e0c9e577
+  metadata.gz: 38eceb446d8dd92f8a0c86d6f7e48fb707babb0c611f437e98322272fcfea124
+  data.tar.gz: ef970349f391a6809cdd91feabe88a5e2beb2c81402ad1ec0ac29ad7bdb0ed1e
 SHA512:
-  metadata.gz: fb35924d7f18579977fa8257a722aa136ca3d9d6a48cb1aecc3aa9f768a4d4b682d5a86c455b634b19d40c6dad9359a54d5b4906ef6952fff8ebc7166c90a808
-  data.tar.gz: abddf838e31d981dc2da2b84bf825cb7981c610da1331a7aba1cf13f2de6e9ce7c644649f586ad6ef9c2630888900f3f8620fec7b47cddbc6b91c927e44c9b72
+  metadata.gz: c0285b5459e65dd7b95766626f6336f6fc92a8a767b78912f9830e8b605beebe5a681f077c7b59b2cc1eb31861854e49ebf89a1372f8ac282fc5a537ad478d54
+  data.tar.gz: 72a905d9d621a80333eeb06a69baa51b435b1efb66acdde4ab51ecf37c6f0aa388ec39af57e0991be1cc1121035212f5a4a9a966a798bfa2794048e8949f33f3

data/CHANGELOG.md

@@ -1,34 +1,38 @@
+## 0.11.2
+  - implemented path_filters to to use path filtering like this **/*.log
+  - implemented debug_until to debug only at the start of a pipeline until it processed enough messages
+
 ## 0.11.1
   - copied changes from irnc fork (danke!)
-  - Fixed trying to load the registry, three time is the charm
+  - fixed trying to load the registry, three time is the charm
   - logs are less chatty, changed info to debug
 
 ## 0.11.0
-  - Implemented start_fresh to skip all previous logs and start monitoring new entries
-  - Fixed the timer, now properly sleep the interval and check again
+  - implemented start_fresh to skip all previous logs and start monitoring new entries
+  - fixed the timer, now properly sleep the interval and check again
   - Work around for a Faraday Middleware v.s. Azure Storage Account bug in follow_redirect
 
 ## 0.10.6
-  - Fixed the rootcause of the checking the codec. Now compare the classname.
+  - fixed the rootcause of the checking the codec. Now compare the classname.
 
 ## 0.10.5
-  - Previous fix broke codec = "line"
+  - previous fix broke codec = "line"
 
 ## 0.10.4
-  - Fixed JSON parsing error for partial files because somehow (logstash 7?) @codec.is_a? doesn't work anymore
+  - fixed JSON parsing error for partial files because somehow (logstash 7?) @codec.is_a? doesn't work anymore
 
 ## 0.10.3
-  - Fixed issue-1 where iplookup confguration was removed, but still used 
+  - fixed issue-1 where iplookup confguration was removed, but still used 
   - iplookup is now done by a separate plugin named logstash-filter-weblookup
 
 ## 0.10.2
   - moved iplookup to own plugin logstash-filter-lookup
 
 ## 0.10.1
-  - Implemented iplookup
-  - Fixed sas tokens (maybe)
-  - Introduced dns_suffix
+  - implemented iplookup
+  - fixed sas tokens (maybe)
+  - introduced dns_suffix
 
 ## 0.10.0
-  - Plugin created with the logstash plugin generator
-  - Reimplemented logstash-input-azureblob with incompatible config and data/registry
+  - plugin created with the logstash plugin generator
+  - reimplemented logstash-input-azureblob with incompatible config and data/registry

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -224,10 +224,15 @@ def run(queue)
                 @processed += wadiislog(queue, name)
             else
                 counter = 0
-                @codec.decode(chunk) do |event|
+                begin
+                  @codec.decode(chunk) do |event|
                     counter += 1
                     decorate(event)
                     queue << event
+                  end
+                rescue Exception => e
+                    @logger.error(@pipe_id+" codec exception: #{e.message} .. will continue and pretend this never happened")
+                    @logger.debug(@pipe_id+" #{chunk}")
                 end
                 @processed += counter
             end
@@ -337,6 +342,7 @@ def list_blobs(fill)
     nextMarker = nil
     for counter in 1..3
       begin
+        loop do
          blobs = @blob_client.list_blobs(container, { marker: nextMarker, prefix: @prefix})
          blobs.each do |blob|
 # FNM_PATHNAME is required so that "**/test" can match "test" at the root folder
@@ -355,6 +361,7 @@ def list_blobs(fill)
          end
          nextMarker = blobs.continuation_token
          break unless nextMarker && !nextMarker.empty?
+        end
       rescue Exception => e
         @logger.error(@pipe_id+" caught: #{e.message} for attempt #{counter} of 3")
 	counter += 1
@@ -386,6 +393,7 @@ def learn_encapsulation
     blob = @blob_client.list_blobs(container, { maxresults: 1, prefix: @prefix }).first
     return if blob.nil?
     blocks = @blob_client.list_blob_blocks(container, blob.name)[:committed]
+    # TODO add check for empty blocks and log error that the header and footer can't be learned and must be set in the config
     @logger.debug(@pipe_id+" using #{blob.name} to learn the json header and tail")
     @head = @blob_client.get_blob(container, blob.name, start_range: 0, end_range: blocks.first.size-1)[1]
     @logger.debug(@pipe_id+" learned header: #{@head}")

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.11.2'
+  s.version       = '0.11.3'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.11.2
+  version: 0.11.3
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-12-20 00:00:00.000000000 Z
+date: 2020-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -113,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.9
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.