logstash-input-azure_blob_storage 0.10.1 → 0.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db216440cf4319f70a5fbdb001a53da72826afdcbce43c04ada28b63c5d9e1f8
-  data.tar.gz: e2e519090c0d67b6b65f4570c34be4f0b02592864459f261c7be613486f2941b
+  metadata.gz: 237b9c5343a6339760b9ab3281a1dfb44f9cc6fcfc826d24928ef95a61984e1e
+  data.tar.gz: 26b2c86c19252cb0c3e681aff153621a8cade7ad47a22c0a6fb093c7764ea79b
 SHA512:
-  metadata.gz: 341abd35cf3b732c1a0bada111cbac79cafa92fb48fd14ad24e0121245c67f1819c1d6e5e4647dda645773231da6aa2a61b7e1aaee4027fa97fe9e857a8a334f
-  data.tar.gz: 93542f740dda404889f623c9a9d460a1ac6f7181ddd6974de8d775e4e29585b8fccd43b9bd05a880250a4da2709121bf5f018f15a8ce49a9d758b1c645482cbf
+  metadata.gz: 88ad3e85f16731b84cd341924f281315d33504c9baef866b91166e9a298b3bc214dde336f621ff8a2de0acf3595a9ffcdfe6ed9b7703cb0795b5b1854b7f7554
+  data.tar.gz: ab0bf519697e96197f1b1ffda10376919a36bfdc9b47d73bdedac7821810b973ade5c5a2acf92b4c6f508c236faffac5fe36ec46c0e6b3a328c34157a25d500e

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.10.2
+  - moved iplookup to own plugin logstash-filter-lookup
+
+## 0.10.1
+  - implemented iplookup
+  - fixed sas tokens (maybe)
+  - introduced dns_suffix
+
 ## 0.10.0
   - Plugin created with the logstash plugin generator
   - Reimplemented logstash-input-azureblob with incompatible config and data/registry

data/README.md

@@ -97,12 +97,6 @@ input {
         prefix => "resourceId=/"
         registry_create_policy => "resume"
         interval => 300
-        iplookup => "http://10.0.0.5:6081/ripe.php?ip="
-        use_redis => true
-        iplist => [
-            "{'ip':'10.0.0.4','netname':'Application Gateway','subnet':'10.0.0.0\/24','hostname':'appgw'}",
-            "{'ip':'36.156.24.96',netname':'China Mobile','subnet':'36.156.0.0\/16','hostname':'bigbadwolf'}"
-        ]
     }
 }
 ```

data/lib/logstash/inputs/azure_blob_storage.rb

@@ -2,13 +2,7 @@
 require "logstash/inputs/base"
 require "stud/interval"
 require 'azure/storage/blob'
-#require 'securerandom'
-#require 'rbconfig'
-#require 'date'
-#require 'json'
-#require 'thread'
-#require 'redis'
-#require 'net/http'
+require 'json'
 
 # This is a logstash input plugin for files in Azure Blob Storage. There is a storage explorer in the portal and an application with the same name https://storageexplorer.com. A storage account has by default a globally unique name, {storageaccount}.blob.core.windows.net which is a CNAME to  Azures blob servers blob.*.store.core.windows.net. A storageaccount has an container and those have a directory and blobs (like files). Blobs have one or more blocks. After writing the blocks, they can be committed. Some Azure diagnostics can send events to an EventHub that can be parse through the plugin logstash-input-azure_event_hubs, but for the events that are only stored in an storage account, use this plugin. The original logstash-input-azureblob from azure-diagnostics-tools is great for low volumes, but it suffers from outdated client, slow reads, lease locking issues and json parse errors.
 # https://azure.microsoft.com/en-us/services/storage/blobs/
@@ -82,33 +76,13 @@ class LogStash::Inputs::AzureBlobStorage < LogStash::Inputs::Base
   # For NSGFLOWLOGS a path starts with "resourceId=/", but this would only be needed to exclude other files that may be written in the same container.
   config :prefix, :validate => :string, :required => false
 
-  # Set the value for the registry file.
-  #
-  # The default, `data/registry`, it contains a Ruby Marshal Serialized Hash of the filename the offset read sofar and the filelength the list time a filelisting was done.
-  config :registry_path, :validate => :string, :required => false, :default => 'data/registry'
-
-  # The default, `resume`, will load the registry offsets and will start processing files from the offsets.
-  # When set to `start_over`, all log files are processed from begining.
-  # when set to `start_fresh`, it will read log files that are created or appended since this start of the pipeline.
-  config :registry_create_policy, :validate => ['resume','start_over','start_fresh'], :required => false, :default => 'resume'
-
-  # Optional to enrich NSGFLOWLOGS with netname and subnet the iplookup value points to a webservice that provides the information in JSON format like this.
-  # {"ip":"8.8.8.8","netname":"Google","subnet":"8.8.8.0\/24","hostname":"google-public-dns-a.google.com"}
-  # In the query parameter has the <ip> tag will be replaced by the IP address to lookup, other parameters are optional and according to your lookup service. 
-  config :iplookup, :validate => :string, :required => false, :default => 'http://127.0.0.1/ripe.php?ip=<ip>&TOKEN=token'
-
-  # Optional array of JSON objects that don't require a lookup
-  config :iplist, :validate => :array, :required => false, :default => ['{"ip":"10.0.0.4","netname":"Application Gateway","subnet":"10.0.0.0\/24","hostname":"appgw"}']
-
-  # Optional Redis IP cache
-  config :use_redis, :validate => :boolean, :required => false, :default => false
-
 
 
 public
 def register
     @pipe_id = Thread.current[:name].split("[").last.split("]").first
-    @logger.info("=== "+config_name+"/"+@pipe_id+"/"+@id[0,6]+" ===")
+    @logger.info("=== "+config_name+" / "+@pipe_id+" / "+@id[0,6]+" ===")
+    #@logger.info("ruby #{ RUBY_VERSION }p#{ RUBY_PATCHLEVEL } / #{Gem.loaded_specs[config_name].version.to_s}")
     @logger.info("Contact me at jan@janmg.com, if something in this plugin doesn't work")
     # TODO: consider multiple readers, so add pipeline @id or use logstash-to-logstash communication?
     # TODO: Implement retry ... Error: Connection refused - Failed to open TCP connection to
@@ -127,9 +101,9 @@ def register
     end
     unless sas_token.nil?
         unless sas_token.value.start_with?('?')
-		conn = "BlobEndpoint=https://#{storageaccount}.#{dns_suffix};SharedAccessSignature=#{sas_token.value}"
+	    conn = "BlobEndpoint=https://#{storageaccount}.#{dns_suffix};SharedAccessSignature=#{sas_token.value}"
         else
-		conn = sas_token.value
+	    conn = sas_token.value
     	end
     end
     unless conn.nil?
@@ -220,7 +194,8 @@ def run(queue)
             end
             if logtype == "nsgflowlog" && @is_json
                 begin
-                    @processed += nsgflowlog(queue, JSON.parse(chunk))
+		    fingjson = JSON.parse(chunk)
+                    @processed += nsgflowlog(queue, fingjson)
                 rescue JSON::ParserError
                     @logger.error(@pipe_id+" parse error on #{res[:nsg]} [#{res[:date]}] offset: #{file[:offset]} length: #{file[:length]}")
                 end
@@ -303,9 +278,11 @@ def nsgflowlog(queue, json)
                   if (record["properties"]["Version"]==2)
                       ev.merge!( {:flowstate => tups[8], :src_pack => tups[9], :src_bytes => tups[10], :dst_pack => tups[11], :dst_bytes => tups[12]} )
                   end
-                  unless iplookup.nil?
-                    ev.merge!(addip(tups[1], tups[2]))
-                  end
+                  # Replaced by new plugin: logstash-filter-lookup
+		  # This caused JSON parse errors since iplookup is now obsolete
+		  #unless iplookup.nil?
+                  #  ev.merge!(addip(tups[1], tups[2]))
+                  #end
                   @logger.trace(ev.to_s)
                   event = LogStash::Event.new('message' => ev.to_json)
                   decorate(event)

data/logstash-input-azure_blob_storage.gemspec

@@ -1,19 +1,19 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-azure_blob_storage'
-  s.version       = '0.10.1'
+  s.version       = '0.10.2'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'This logstash plugin reads and parses data from Azure Storage Blobs.'
   s.description   = <<-EOF
  This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial file reads and eliminating a delay when rebuilding the registry.
 
- The logstash pipeline configuration would look like this
-  input {
-    azure_blob_storage {
-        storageaccount => "yourstorageaccountname"
-        access_key => "Ba5e64c0d3=="
-        container => "insights-logs-networksecuritygroupflowevent"
-    }
-  }
+ The minimal logstash pipeline configuration would look like this
+> input {
+>   azure_blob_storage {
+>       storageaccount => "yourstorageaccountname"
+>       access_key => "Ba5e64c0d3=="
+>       container => "insights-logs-networksecuritygroupflowevent"
+>   }
+> }
 EOF
   s.homepage      = 'https://github.com/janmg/logstash-input-azure_blob_storage'
   s.authors       = ['Jan Geertsma']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azure_blob_storage
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.10.2
 platform: ruby
 authors:
 - Jan Geertsma
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-22 00:00:00.000000000 Z
+date: 2019-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -69,34 +69,34 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: |2
    This gem is a Logstash plugin. It reads and parses data from Azure Storage Blobs. The azure_blob_storage is a reimplementation to replace azureblob from azure-diagnostics-tools/Logstash. It can deal with larger volumes and partial file reads and eliminating a delay when rebuilding the registry.
 
-   The logstash pipeline configuration would look like this
-    input {
-      azure_blob_storage {
-          storageaccount => "yourstorageaccountname"
-          access_key => "Ba5e64c0d3=="
-          container => "insights-logs-networksecuritygroupflowevent"
-      }
-    }
+   The minimal logstash pipeline configuration would look like this
+  > input {
+  >   azure_blob_storage {
+  >       storageaccount => "yourstorageaccountname"
+  >       access_key => "Ba5e64c0d3=="
+  >       container => "insights-logs-networksecuritygroupflowevent"
+  >   }
+  > }
 email: jan@janmg.com
 executables: []
 extensions: []
@@ -133,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 2.7.9
 signing_key:
 specification_version: 4
 summary: This logstash plugin reads and parses data from Azure Storage Blobs.