logstash-input-azure_blob_storage 0.11.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb93fc423babf6bc4cd7b13b1280a27ea6156fa2aeebe69ab172d8e925940d2c
-  data.tar.gz: ae92a22e56d87cc9d4d0f0b615460a8da33afb822aaa3f0d109cd3b86137ee53
+  metadata.gz: 5fb68f13f46e7a0455fe4ffd3f6c9e04b136611e01504310bd739bbc6813c6f6
+  data.tar.gz: 3f818813b0b45acac96edb34a4948d01c234946fb2580eefe5ece8e43240c0c1
 SHA512:
-  metadata.gz: f5a311f322b04740a98182271e3074171feea7b7e899e3ec712b457182d91f34033cc73b1219042bad727f0c6566b1c3cf0ce362e5cc9d2b1e4d09a2029d5456
-  data.tar.gz: 53072a976feddc171ad02960fdcec6612099caaade967c28a501ba1ca413f1dccdc071e050e322eefa84a266b0c9a9ed487a6d98a36c62480e573421b2fc27b7
+  metadata.gz: b596bbfc6a1e3400c33e54bbfa4adb753ea1c6593ae647da221368a089b25cd650856d4abb78c5f39ae39df67387b3de938962d63a90e06d2b54164599ced0a9
+  data.tar.gz: 6c0eb3959fa0f393f63c0697f26d49b01280604e2443b8c0a17342d768f4c1a9402e4c8f658462cdd1d76cc11fa7a654bb36052ff05fdbec775050ad33539a1c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.11.0
+  - Implemented start_fresh to skip all previous logs and start monitoring new entries
+  - Fixed the timer, now properly sleep the interval and check again
+  - Work around for a Faraday Middleware v.s. Azure Storage Account bug in follow_redirect
+
 ## 0.10.6
   - Fixed the rootcause of the checking the codec. Now compare the classname.
 

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -91,8 +91,6 @@ def register
     @processed = 0
     @regsaved = @processed
 
-    #@buffer = FileWatch::BufferedTokenizer.new('\n')
-
     # Try in this order to access the storageaccount
     # 1. storageaccount / sas_token
     # 2. connection_string
@@ -120,22 +118,22 @@ def register
 
     @registry = Hash.new
     if registry_create_policy == "resume"
-      begin
-        @logger.info(@pipe_id+" resuming from registry")
-        @registry = Marshal.load(@blob_client.get_blob(container, registry_path)[1])
-        #[0] headers [1] responsebody
-      rescue
-        @registry.clear
-        @logger.error(@pipe_id+" loading registry failed, starting over")
+     @logger.info(@pipe_id+" resuming from registry")
+     for counter in 0..3
+       begin
+          @registry = Marshal.load(@blob_client.get_blob(container, registry_path)[1])
+          #[0] headers [1] responsebody
+        rescue Exception => e
+          @logger.error(@pipe_id+" caught: #{e.message}")
+          @registry.clear
+          @logger.error(@pipe_id+" loading registry failed, starting over")
+        end
       end
     end
     # read filelist and set offsets to file length to mark all the old files as done
     if registry_create_policy == "start_fresh"
         @logger.info(@pipe_id+" starting fresh")
         @registry = list_blobs(true)
-        #tempreg.each do |name, file|
-        #   @registry.store(name, { :offset => file[:length], :length => file[:length] })
-        #end
     end
 
     @is_json = false
@@ -168,8 +166,8 @@ def run(queue)
     # we can abort the loop if stop? becomes true
     while !stop?
         chrono = Time.now.to_i
-        # load te registry, compare it's offsets to file list, set offset to 0 for new files, process the whole list and if finished within the interval wait for next loop, 
-        # TODO: sort by timestamp
+        # load the registry, compare it's offsets to file list, set offset to 0 for new files, process the whole list and if finished within the interval wait for next loop, 
+        # TODO: sort by timestamp ?
         #filelist.sort_by(|k,v|resource(k)[:date])
 	worklist.clear
 	filelist.clear
@@ -192,7 +190,7 @@ def run(queue)
         # This would be ideal for threading since it's IO intensive, would be nice with a ruby native ThreadPool
         worklist.each do |name, file|
             #res = resource(name)
-            @logger.info(@pipe_id+" processing #{name} from #{file[:offset]} to #{file[:length]}")
+            @logger.debug(@pipe_id+" processing #{name} from #{file[:offset]} to #{file[:length]}")
             size = 0
             if file[:offset] == 0
                 chunk = full_read(name)
@@ -224,6 +222,8 @@ def run(queue)
                 @processed += counter
             end
             @registry.store(name, { :offset => size, :length => file[:length] })
+            # TODO add input plugin option to prevent connection cache
+            @blob_client.client.reset_agents!
 	    #@logger.info(@pipe_id+" name #{name} size #{size} len #{file[:length]}")
             # if stop? good moment to stop what we're doing
             if stop?
@@ -246,7 +246,9 @@ end
 def stop
     save_registry(@registry)
 end
-
+def close
+    save_registry(@registry)
+end
 
 
 private
@@ -371,7 +373,6 @@ def learn_encapsulation
     # From one file, read first block and last block to learn head and tail
     # If the blobstorage can't be found, an error from farraday middleware will come with the text
     # org.jruby.ext.set.RubySet cannot be cast to class org.jruby.RubyFixnum
-    # implement options ... prefix may ot exist!
     blob = @blob_client.list_blobs(container, { maxresults: 1, prefix: @prefix }).first
     return if blob.nil?
     blocks = @blob_client.list_blob_blocks(container, blob.name)[:committed]

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.11.0'
+  s.version       = '0.11.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.11.1
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-15 00:00:00.000000000 Z
+date: 2019-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -112,8 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.9
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.