logstash-input-azure_blob_storage 0.11.7 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0dd48413c8fc381dc144c1a4a58c82906533e4f94b36d631c597ecc766aa8edf
-  data.tar.gz: 3fb23ac270d539092ca52d73418c710e6d12816635b066da004728bdaef3cc9b
+  metadata.gz: da5b56c9fb4733d62604b7753d2e08e3d24100915f58d4f374ab6b6573d27408
+  data.tar.gz: 26f5a19de67615eeb27d302e0b0768fc0b3199f33808182b32a3aeed422b2599
 SHA512:
-  metadata.gz: 84c46edd2afbfe316c2fd5a3b8601f8841308270c3050425a62def1305065aae7868f8be14a98fcc1a7b98f3ccfeee3c9f8a9f9652bf7e799f8d8c5ba1016334
-  data.tar.gz: f2a2a27e068f8d384be829abde10935d0cc323da3eb943264ce05e11d9ee1f383fcfac47cbad51a909623638477b0bef5323d2a5b2def1d8666444d08511aa24
+  metadata.gz: 37f134a2ef4cb0cae2a24e38a3299687f56bc2bf7d98f56976d2bba605b356caec39022c33d18fb58d51933eaa0a3d747c0c4cfacfbe4c0ce6c029b0ff6ddd78
+  data.tar.gz: abd3a6803577db03c42eebf69795eded5e948e48a38363fa81b566c2ecc494d9313d07e280569ab424c561a91c6a08a00d1cfa6bc7674fd84b137e08b4af75ef

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## 0.12.3
+  - Fixed repetative processing
+  - Replaced newreg with registry cleanup, using newreg to replace old registry worked in 0.11, but not in .12
+  - Implemented Mutex for the save_registry thread. Also marshal @registry before thread start for better thread safety
+
+## 0.12.2
+  - Fixed the exception handling, not trying to print how many JSON fields there are while catching the exception
+
+## 0.12.1
+  - Catch NSGFLOW logs when the JSON parsing somehow failed
+
+## 0.12.0
+  - version 2 of azure-storage
+  - saving current files registry, not keeping historical files
+
+## 0.11.7
+  - implemented skip_learning
+  - start ignoring failed files and not retry
+
 ## 0.11.6
   - fix in json head and tail learning the max_results
   - broke out connection setup in order to call it again if connection exceptions come

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -96,10 +96,6 @@ config :prefix, :validate => :string, :required => false
 
 config :path_filters, :validate => :array, :default => ['**/*'], :required => false
 
-# TODO: Other feature requests
-# show file path in logger
-# add filepath as part of log message
-# option to keep registry on local disk
 
 
 public
@@ -107,6 +103,7 @@ def register
     @pipe_id = Thread.current[:name].split("[").last.split("]").first
     @logger.info("=== #{config_name} #{Gem.loaded_specs["logstash-input-"+config_name].version.to_s} / #{@pipe_id} / #{@id[0,6]} / ruby #{ RUBY_VERSION }p#{ RUBY_PATCHLEVEL } ===")
     @logger.info("If this plugin doesn't work, please raise an issue in https://github.com/janmg/logstash-input-azure_blob_storage")
+    @busy_writing_registry = Mutex.new
     # TODO: consider multiple readers, so add pipeline @id or use logstash-to-logstash communication?
     # TODO: Implement retry ... Error: Connection refused - Failed to open TCP connection to
 end
@@ -152,7 +149,7 @@ def run(queue)
     # read filelist and set offsets to file length to mark all the old files as done
     if registry_create_policy == "start_fresh"
         @registry = list_blobs(true)
-	save_registry(@registry)
+	save_registry()
 	@logger.info("starting fresh, writing a clean registry to contain #{@registry.size} blobs/files")
     end
 
@@ -178,7 +175,6 @@ def run(queue)
         @logger.info("head will be: #{@head} and tail is set to #{@tail}")
     end
 
-    newreg   = Hash.new
     filelist = Hash.new
     worklist = Hash.new
     @last = start = Time.now.to_i
@@ -197,7 +193,6 @@ def run(queue)
         #filelist.sort_by(|k,v|resource(k)[:date])
 	worklist.clear
 	filelist.clear
-        newreg.clear
 
 	# Listing all the files
         filelist = list_blobs(false)
@@ -208,15 +203,24 @@ def run(queue)
             rescue
                 off = 0
             end
-            newreg.store(name, { :offset => off, :length => file[:length] })
+            @registry.store(name, { :offset => off, :length => file[:length] })
 	    if (@debug_until > @processed) then @logger.info("2: adding offsets: #{name} #{off} #{file[:length]}") end
 	end
         # size nilClass when the list doesn't grow?!
+
+        # clean registry of files that are not in the filelist
+        @registry.each do |name,file|
+            unless filelist.include?(name)
+                @registry.delete(name)
+                if (@debug_until > @processed) then @logger.info("purging #{name}") end
+            end
+        end
+
         # Worklist is the subset of files where the already read offset is smaller than the file size
-	worklist.clear
+        worklist.clear
         chunk = nil
 
-	worklist = newreg.select {|name,file| file[:offset] < file[:length]}
+	worklist = @registry.select {|name,file| file[:offset] < file[:length]}
 	if (worklist.size > 4) then @logger.info("worklist contains #{worklist.size} blobs") end
 
         # Start of processing
@@ -235,7 +239,8 @@ def run(queue)
                         chunk = full_read(name)
                         size=chunk.size
                     rescue Exception => e
-                        @logger.error("Failed to read #{name} because of: #{e.message} .. will continue, set file as read and pretend this never happened")
+                        # Azure::Core::Http::HTTPError / undefined method `message='
+                      @logger.error("Failed to read #{name} ... will continue, set file as read and pretend this never happened")
                         @logger.error("#{size} size and #{file[:length]} file length")
                         size = file[:length]
                     end 
@@ -274,12 +279,12 @@ def run(queue)
                     decorate(event)
                     queue << event
                   end
+                @processed += counter
                 rescue Exception => e
                     @logger.error("codec exception: #{e.message} .. will continue and pretend this never happened")
                     @registry.store(name, { :offset => file[:length], :length => file[:length] })
                     @logger.debug("#{chunk}")
                 end
-                @processed += counter
             end
             @registry.store(name, { :offset => size, :length => file[:length] })
             # TODO add input plugin option to prevent connection cache
@@ -290,14 +295,14 @@ def run(queue)
                 return
             end
 	    if ((Time.now.to_i - @last) > @interval)
-                save_registry(@registry)
+                save_registry()
             end
           end
         end
 	# The files that got processed after the last registry save need to be saved too, in case the worklist is empty for some intervals.
         now = Time.now.to_i
 	if ((now - @last) > @interval)
-            save_registry(@registry)
+            save_registry()
         end
         sleeptime = interval - ((now - start) % interval)
 	if @debug_timer
@@ -308,10 +313,10 @@ def run(queue)
 end
 
 def stop
-    save_registry(@registry)
+    save_registry()
 end
 def close
-    save_registry(@registry)
+    save_registry()
 end
 
 
@@ -388,6 +393,7 @@ end
 
 def nsgflowlog(queue, json, name)
     count=0
+    begin
     json["records"].each do |record|
       res = resource(record["resourceId"])
       resource = { :subscription => res[:subscription], :resourcegroup => res[:resourcegroup], :nsg => res[:nsg] }
@@ -417,6 +423,9 @@ def nsgflowlog(queue, json, name)
           end
       end
     end
+    rescue Exception => e
+      @logger.error("NSG Flowlog problem for #{name} and error message #{e.message}")
+    end
     return count 
 end
 
@@ -485,30 +494,35 @@ def try_list_blobs(fill)
 end
 
 # When events were processed after the last registry save, start a thread to update the registry file.
-def save_registry(filelist)
-    # Because of threading, processed values and regsaved are not thread safe, they can change as instance variable @! Most of the time this is fine because the registry is the last resort, but be careful about corner cases!
+def save_registry()
     unless @processed == @regsaved
-        @regsaved = @processed
-        unless (@busy_writing_registry)
-	Thread.new {
-            begin
-                @busy_writing_registry = true
-                unless (@registry_local_path)
-                    @blob_client.create_block_blob(container, registry_path, Marshal.dump(filelist))
-                    @logger.info("processed #{@processed} events, saving #{filelist.size} blobs and offsets to remote registry #{registry_path}")
-                else
-                    File.open(@registry_local_path+"/"+@pipe_id, 'w') { |file| file.write(Marshal.dump(filelist)) }
-                    @logger.info("processed #{@processed} events, saving #{filelist.size} blobs and offsets to local registry #{registry_local_path+"/"+@pipe_id}")
-                end
-                @busy_writing_registry = false
-                @last = Time.now.to_i
-            rescue
-                @logger.error("Oh my, registry write failed, do you have write access?")
-            end
+      unless (@busy_writing_registry.locked?)
+        # deep_copy hash, to save the registry independant from the variable for thread safety
+        # if deep_clone uses Marshall to do a copy, 
+        regdump = Marshal.dump(@registry)
+        regsize = @registry.size
+        Thread.new {
+        begin
+          @busy_writing_registry.lock
+          unless (@registry_local_path)
+              @blob_client.create_block_blob(container, registry_path, regdump)
+              @logger.info("processed #{@processed} events, saving #{regsize} blobs and offsets to remote registry #{registry_path}")
+          else
+              File.open(@registry_local_path+"/"+@pipe_id, 'w') { |file| file.write(regdump) }
+              @logger.info("processed #{@processed} events, saving #{regsize} blobs and offsets to local registry #{registry_local_path+"/"+@pipe_id}")
+          end
+          @last = Time.now.to_i
+          @regsaved = @processed
+          rescue Exception => e
+              @logger.error("Oh my, registry write failed")
+              @logger.error("#{e.message}")
+          ensure
+              @busy_writing_registry.unlock
+          end
         }
-        else
-            @logger.info("Skipped writing the registry because previous write still in progress, it just takes long or may be hanging!")
-        end
+      else
+        @logger.info("Skipped writing the registry because previous write still in progress, it just takes long or may be hanging!")
+      end
     end
 end
 

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.11.7'
+  s.version       = '0.12.3'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF
@@ -20,8 +20,8 @@ EOF
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.1'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
   s.add_runtime_dependency 'stud', '~> 0.0.23'
-  s.add_runtime_dependency 'azure-storage-blob', '~> 1.1'
-  #s.add_development_dependency 'logstash-devutils', '~> 2'
+  s.add_runtime_dependency 'azure-storage-blob', '~> 2', '>= 2.0.3'
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/azure_blob_storage_spec.rb

@@ -8,6 +8,10 @@ describe LogStash::Inputs::AzureBlobStorage do
     let(:config) { { "interval" => 100 } }
   end
 
+  def test_val
+      assert_qual('abc', val('letters=abc'))
+  end
+
   def test_helper_methodes 
       assert_equal('b', AzureBlobStorage.val('a=b'))
       assert_equal('whatever', AzureBlobStorage.strip_comma(',whatever'))

data/spec/inputs/test.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/test"
+
+describe LogStash::Inputs::Test do
+
+  it_behaves_like "an interruptible input plugin" do
+    let(:config) { { "interval" => 100 } }
+  end
+
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.11.7
+  version: 0.12.3
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2022-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.0'
   name: logstash-core-plugin-api
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -31,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.23
   name: stud
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -43,15 +43,35 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
   name: azure-storage-blob
-  type: :runtime
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: " This gem is a Logstash plugin. It reads and parses data from Azure\
   \ Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob\
   \ from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial\
@@ -71,6 +91,7 @@ files:
 - lib/logstash/inputs/azure_blob_storage.rb
 - logstash-input-azure_blob_storage.gemspec
 - spec/inputs/azure_blob_storage_spec.rb
+- spec/inputs/test.rb
 homepage: https://github.com/janmg/logstash-input-azure_blob_storage
 licenses:
 - Apache-2.0
@@ -92,9 +113,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.
 test_files:
 - spec/inputs/azure_blob_storage_spec.rb
+- spec/inputs/test.rb