logstash-input-azure_blob_storage 0.10.5 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 781172a19e14c687455ae28a4a306b04154336b4a1f250a524e711dde3c7d751
-  data.tar.gz: c38c2cc4eb832285e3db7e3146d69fc4bf26b5c04e2981229a2b5bb330ec5345
+  metadata.gz: 409d47c61b8f38963281ca655d7664bba48c6d311adbcbbbb6a4e2e5d0356363
+  data.tar.gz: 1648a2174502fb84d015bcb1b76f192c7c7583b3e71ce5ec738ee20da12d0f8b
 SHA512:
-  metadata.gz: b0a396dfc88d41be843a562b2473c2e8b34a04ae7532d67be215c4d64130a23a9afa63e875c79d48080f3ba887acc2996d29955c3fff7e7e94ac7cee65830873
-  data.tar.gz: 26d6884ac46ab287e9067bbf49fa597281cc8b8901302c30bf850daba582de7ba07333a79570e2bffc1dce84e2b02bdb47a99706cc4361969f5380a1ede38a27
+  metadata.gz: '0795b03a70aee648d1cd7c951c1f3bc6fc6d8bc3989098ed4982e1011f66cca7a83c77a366351db00254cf3b2ee3b3b1389d8db0f2e9fc2d6424e7218c470a06'
+  data.tar.gz: 85e0d362dfacc7369fdb04e4dc47ea3754ba64c175029bc8c46e0ce14fd30ad2a0660f9b051f0b93eeb5ddffd6b6f06a111819116cbb7e07f20d9e310f418e2d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.10.6
+  - Fixed the rootcause of the checking the codec. Now compare the classname.
+
 ## 0.10.5
   - Previous fix broke codec = "line"
 

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -91,6 +91,8 @@ def register
     @processed = 0
     @regsaved = @processed
 
+    @buffer = FileWatch::BufferedTokenizer.new('\n')
+
     # Try in this order to access the storageaccount
     # 1. storageaccount / sas_token
     # 2. connection_string
@@ -134,14 +136,10 @@ def register
 
     @is_json = false
     begin
-      if @codec.is_a?(LogStash::Codecs::JSON)
+      if @codec.class.name.eql?("LogStash::Codecs::JSON")
         @is_json = true 
       end
-    rescue
-        @logger.debug(@pipe_id+" Rescue from uninitialized constant ...")
-	# how can you elegantly check the codec type in logstash? anyway, not worth crashing over since is_json is already set to false by default
     end
-    @logger.debug(@pipe_id+" is_json is set to: #{@is_json} because it is a #{@codec}")
     @head = ''
     @tail = ''
     # if codec=json sniff one files blocks A and Z to learn file_head and file_tail
@@ -153,15 +151,17 @@ def register
         if file_tail
            @tail = file_tail
         end
+        @logger.info(@pipe_id+" head will be: #{@head} and tail is set to #{@tail}")
     end
-    @logger.info(@pipe_id+" head will be: #{@head} and tail is set to #{@tail}")
 end # def register
 
+
+
 def run(queue)
     filelist = Hash.new
 
-      # we can abort the loop if stop? becomes true
-      while !stop?
+    # we can abort the loop if stop? becomes true
+    while !stop?
         chrono = Time.now.to_i
         # load te registry, compare it's offsets to file list, set offset to 0 for new files, process the whole list and if finished within the interval wait for next loop, 
         # TODO: sort by timestamp
@@ -183,14 +183,13 @@ def run(queue)
                 file[:length]=chunk.size
             else
                 chunk = partial_read_json(name, file[:offset], file[:length])
-                # This only applies to NSG!
-		@logger.info(@pipe_id+" partial file #{res[:nsg]} [#{res[:date]}]")
-		 @logger.info(@pipe_id+" partial file #{name}")
+                @logger.debug(@pipe_id+" partial file #{name} from #{file[:offset]} to #{file[:length]}")
             end
             if logtype == "nsgflowlog" && @is_json
                 begin
 		    fingjson = JSON.parse(chunk)
                     @processed += nsgflowlog(queue, fingjson)
+                    @logger.debug(@pipe_id+" Processed #{res[:nsg]} [#{res[:date]}] #{@processed} events")
                 rescue JSON::ParserError
                     @logger.error(@pipe_id+" parse error on #{res[:nsg]} [#{res[:date]}] offset: #{file[:offset]} length: #{file[:length]}")
                 end
@@ -198,24 +197,24 @@ def run(queue)
 	    elsif logtype == "wadiis" && !@is_json
                 @processed += wadiislog(queue, file[:name])
             else
+                counter = 0
                 @codec.decode(chunk) do |event|
+                    counter += 1
                     decorate(event)
                     queue << event
                 end
-                @processed += 1
+                @processed += counter
             end
-	    # This only applies to NSG!
-            @logger.debug(@pipe_id+" Processed #{res[:nsg]} [#{res[:date]}] #{@processed} events")
             @registry.store(name, { :offset => file[:length], :length => file[:length] })
             # if stop? good moment to stop what we're doing
             if stop?
                 return
             end
-            # save the registry regularly
+            # save the registry past the regular intervals
             now = Time.now.to_i
             if ((now - chrono) > interval)
                 save_registry(@registry)
-                chrono = now
+		chrono =+ interval
             end
         end
         # Save the registry and sleep until the remaining polling interval is over
@@ -223,9 +222,7 @@ def run(queue)
         sleeptime = interval - (Time.now.to_i - chrono)
         Stud.stoppable_sleep(sleeptime) { stop? }
     end
-
-    #  event = LogStash::Event.new("message" => @message, "host" => @host)
-  end # def run
+end
 
 def stop
     save_registry(@registry)
@@ -274,11 +271,6 @@ def nsgflowlog(queue, json)
                   if (record["properties"]["Version"]==2)
                       ev.merge!( {:flowstate => tups[8], :src_pack => tups[9], :src_bytes => tups[10], :dst_pack => tups[11], :dst_bytes => tups[12]} )
                   end
-                  # Replaced by new plugin: logstash-filter-lookup
-		  # This caused JSON parse errors since iplookup is now obsolete
-		  #unless iplookup.nil?
-                  #  ev.merge!(addip(tups[1], tups[2]))
-                  #end
                   @logger.trace(ev.to_s)
                   event = LogStash::Event.new('message' => ev.to_json)
                   decorate(event)
@@ -354,13 +346,13 @@ def learn_encapsulation
     blob = @blob_client.list_blobs(container, { maxresults: 1, prefix: @prefix }).first
     return if blob.nil?
     blocks = @blob_client.list_blob_blocks(container, blob.name)[:committed]
-    @logger.info(@pipe_id+" using #{blob.name} to learn the json header and tail")
+    @logger.debug(@pipe_id+" using #{blob.name} to learn the json header and tail")
     @head = @blob_client.get_blob(container, blob.name, start_range: 0, end_range: blocks.first.size-1)[1]
-    @logger.info(@pipe_id+" learned header: #{@head}")
+    @logger.debug(@pipe_id+" learned header: #{@head}")
     length = blob.properties[:content_length].to_i
     offset = length - blocks.last.size
     @tail = @blob_client.get_blob(container, blob.name, start_range: offset, end_range: length-1)[1]
-    @logger.info(@pipe_id+" learned tail: #{@tail}")
+    @logger.debug(@pipe_id+" learned tail: #{@tail}")
 end
 
 def resource(str)

data/logstash-input-azure_blob_storage.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.10.5'
+  s.version       = '0.10.6'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF
@@ -21,7 +21,6 @@ EOF
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_runtime_dependency 'logstash-codec-plain', '~> 3.0'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_runtime_dependency 'azure-storage-blob', '~> 1.0'
   s.add_development_dependency 'logstash-devutils', '~> 1.0', '>= 1.0.0'

data/spec/inputs/azure_blob_storage_spec.rb

@@ -9,10 +9,10 @@ describe LogStash::Inputs::AzureBlobStorage do
   end
 
   def test_helper_methodes 
-      assert_equal('b', AzureBlobStorage.val('a=b')
-      assert_equal('whatever', AzureBlobStorage.strip_comma(',whatever')
-      assert_equal('whatever', AzureBlobStorage.strip_comma('whatever,')
-      assert_equal('whatever', AzureBlobStorage.strip_comma(',whatever,')
-      assert_equal('whatever', AzureBlobStorage.strip_comma('whatever')
+      assert_equal('b', AzureBlobStorage.val('a=b'))
+      assert_equal('whatever', AzureBlobStorage.strip_comma(',whatever'))
+      assert_equal('whatever', AzureBlobStorage.strip_comma('whatever,'))
+      assert_equal('whatever', AzureBlobStorage.strip_comma(',whatever,'))
+      assert_equal('whatever', AzureBlobStorage.strip_comma('whatever'))
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.6
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-08 00:00:00.000000000 Z
+date: 2019-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  name: logstash-codec-plain
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: