logstash-input-azure_blob_storage 0.11.1 → 0.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fb68f13f46e7a0455fe4ffd3f6c9e04b136611e01504310bd739bbc6813c6f6
-  data.tar.gz: 3f818813b0b45acac96edb34a4948d01c234946fb2580eefe5ece8e43240c0c1
+  metadata.gz: b721a6aa74f4e9df285f62f47efa42112e540d9836391b31e74daf6544e1087d
+  data.tar.gz: 5d22a077d53698807a51dde75ac6c7deb273f0fe68d7ea05a46651b5e0c9e577
 SHA512:
-  metadata.gz: b596bbfc6a1e3400c33e54bbfa4adb753ea1c6593ae647da221368a089b25cd650856d4abb78c5f39ae39df67387b3de938962d63a90e06d2b54164599ced0a9
-  data.tar.gz: 6c0eb3959fa0f393f63c0697f26d49b01280604e2443b8c0a17342d768f4c1a9402e4c8f658462cdd1d76cc11fa7a654bb36052ff05fdbec775050ad33539a1c
+  metadata.gz: fb35924d7f18579977fa8257a722aa136ca3d9d6a48cb1aecc3aa9f768a4d4b682d5a86c455b634b19d40c6dad9359a54d5b4906ef6952fff8ebc7166c90a808
+  data.tar.gz: abddf838e31d981dc2da2b84bf825cb7981c610da1331a7aba1cf13f2de6e9ce7c644649f586ad6ef9c2630888900f3f8620fec7b47cddbc6b91c927e44c9b72

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.11.1
+  - copied changes from irnc fork (danke!)
+  - Fixed trying to load the registry, three time is the charm
+  - logs are less chatty, changed info to debug
+
 ## 0.11.0
   - Implemented start_fresh to skip all previous logs and start monitoring new entries
   - Fixed the timer, now properly sleep the interval and check again

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -55,6 +55,9 @@ config :registry_create_policy, :validate => ['resume','start_over','start_fresh
 # Z00000000000000000000000000000000 2     ]}
 config :interval, :validate => :number, :default => 60
 
+# debug_until will for a maximum amount of processed messages shows 3 types of log printouts including processed filenames. This is a lightweight alternative to switching the loglevel from info to debug or even trace 
+config :debug_until, :validate => :number, :default => 0, :required => false
+
 # WAD IIS Grok Pattern
 #config :grokpattern, :validate => :string, :required => false, :default => '%{TIMESTAMP_ISO8601:log_timestamp} %{NOTSPACE:instanceId} %{NOTSPACE:instanceId2} %{IPORHOST:ServerIP} %{WORD:httpMethod} %{URIPATH:requestUri} %{NOTSPACE:requestQuery} %{NUMBER:port} %{NOTSPACE:username} %{IPORHOST:clientIP} %{NOTSPACE:httpVersion} %{NOTSPACE:userAgent} %{NOTSPACE:cookie} %{NOTSPACE:referer} %{NOTSPACE:host} %{NUMBER:httpStatus} %{NUMBER:subresponse} %{NUMBER:win32response} %{NUMBER:sentBytes:int} %{NUMBER:receivedBytes:int} %{NUMBER:timeTaken:int}'
 
@@ -76,6 +79,12 @@ config :file_tail, :validate => :string, :required => false, :default => ']}'
 # For NSGFLOWLOGS a path starts with "resourceId=/", but this would only be needed to exclude other files that may be written in the same container.
 config :prefix, :validate => :string, :required => false
 
+config :path_filters, :validate => :array, :default => ['**/*'], :required => false
+
+# TODO: Other feature requests
+# show file path in logger
+# add filepath as part of log message
+# option to keep registry on local disk
 
 
 public
@@ -119,14 +128,14 @@ def register
     @registry = Hash.new
     if registry_create_policy == "resume"
      @logger.info(@pipe_id+" resuming from registry")
-     for counter in 0..3
+     for counter in 1..3
        begin
           @registry = Marshal.load(@blob_client.get_blob(container, registry_path)[1])
           #[0] headers [1] responsebody
         rescue Exception => e
           @logger.error(@pipe_id+" caught: #{e.message}")
           @registry.clear
-          @logger.error(@pipe_id+" loading registry failed, starting over")
+          @logger.error(@pipe_id+" loading registry failed for attempt #{counter} of 3")
         end
       end
     end
@@ -182,6 +191,7 @@ def run(queue)
                 off = 0
             end
             newreg.store(name, { :offset => off, :length => file[:length] })
+	    if (@debug_until > @processed) then @logger.info("2: adding offsets: #{name} #{off} #{file[:length]}") end
 	end
         
         # Worklist is the subset of files where the already read offset is smaller than the file size
@@ -190,14 +200,14 @@ def run(queue)
         # This would be ideal for threading since it's IO intensive, would be nice with a ruby native ThreadPool
         worklist.each do |name, file|
             #res = resource(name)
-            @logger.debug(@pipe_id+" processing #{name} from #{file[:offset]} to #{file[:length]}")
+            if (@debug_until > @processed) then @logger.info("3: processing #{name} from #{file[:offset]} to #{file[:length]}") end
             size = 0
             if file[:offset] == 0
                 chunk = full_read(name)
                 size=chunk.size
             else
                 chunk = partial_read_json(name, file[:offset], file[:length])
-                @logger.debug(@pipe_id+" partial file #{name} from #{file[:offset]} to #{file[:length]}")
+                @logger.info(@pipe_id+" partial file #{name} from #{file[:offset]} to #{file[:length]}")
             end
             if logtype == "nsgflowlog" && @is_json
                 res = resource(name)
@@ -325,28 +335,28 @@ end
 def list_blobs(fill)
     files = Hash.new
     nextMarker = nil
-    counter = 0
-    loop do
+    for counter in 1..3
       begin
-         if (counter > 10)
-             @logger.error(@pipe_id+" lets try again for the 10th time, why don't faraday and azure storage accounts not play nice together? it has something to do with follow_redirect and a missing authorization header?")
-         end
          blobs = @blob_client.list_blobs(container, { marker: nextMarker, prefix: @prefix})
          blobs.each do |blob|
-             # exclude the registry itself
-             unless blob.name == registry_path
+# FNM_PATHNAME is required so that "**/test" can match "test" at the root folder
+# FNM_EXTGLOB allows you to use "test{a,b,c}" to match either "testa", "testb" or "testc" (closer to shell behavior)
+           unless blob.name == registry_path
+	     if @path_filters.any? {|path| File.fnmatch?(path, blob.name, File::FNM_PATHNAME | File::FNM_EXTGLOB)}
                  length = blob.properties[:content_length].to_i
-		 offset = 0
+                 offset = 0
                  if fill
                      offset = length
-		 end
+                 end
                  files.store(blob.name, { :offset => offset, :length => length })
+		 if (@debug_until > @processed) then @logger.info("1: list_blobs #{blob.name} #{offset} #{length}") end
              end
+           end
          end
          nextMarker = blobs.continuation_token
          break unless nextMarker && !nextMarker.empty?
       rescue Exception => e
-        @logger.error(@pipe_id+" caught: #{e.message}")
+        @logger.error(@pipe_id+" caught: #{e.message} for attempt #{counter} of 3")
 	counter += 1
       end
     end

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.11.1'
+  s.version       = '0.11.2'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.11.2
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2019-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -112,7 +112,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubyforge_project:
+rubygems_version: 2.7.9
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.