logstash-input-azurewadtable 0.9.9 → 0.9.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e1a247c141b52ee483519dee54974046c6e88dc0
-  data.tar.gz: 9b6375c87ab768b39891f626ad4875b2abadd7d9
+  metadata.gz: c27925f10618e725258c57b9afb50b9f8c25689c
+  data.tar.gz: 3f4a6dc33cbf979edeaf97b6696470b22bd7f5ff
 SHA512:
-  metadata.gz: 7b4fd596fbd153b6cfb3beedf662b9f205514ae3417ccc56bb68f491baad3e2d6977c43896db3959e2adb3bf08b56dc705f25f53e3ed09fffbab7f439cf1ebff
-  data.tar.gz: 21a259d435e4d5ced92b9efb77d8fb76d0f49e54dd52b2a411998d8c73dc02f64ca24a365fea0a1c89d7e138b9f448192747c97abeec1d094e3ffffba8a051d0
+  metadata.gz: 11ac6192c5c76e265bab0412520b795e898e28e5861aeacdebdfd05634da3c718bc4b4aa079f4706959fe63c4ddca2191ae21505785067516934a1028153947c
+  data.tar.gz: 1581ff53ae508173137aa0ccb8ad96fc5c224f4385bd6b7f1535362196d4c44c050eef4aa4b676d3df162141941b85ff2ec94aca18a62626fa31149f3087be2b

data/CHANGELOG.md

@@ -1,7 +1,7 @@
-## 2016.06.27
-* Added support for setting Azure service endpoint in the configuration.
-
-## 2016.05.02
-* Made the plugin to respect Logstash shutdown signal.
-* Updated the *logstash-core* runtime dependency requirement to '~> 2.0'.
+## 2016.06.27
+* Added support for setting Azure service endpoint in the configuration.
+
+## 2016.05.02
+* Made the plugin to respect Logstash shutdown signal.
+* Updated the *logstash-core* runtime dependency requirement to '~> 2.0'.
 * Updated the *logstash-devutils* development dependency requirement to '>= 0.0.16'

data/Gemfile

@@ -1,2 +1,2 @@
-source 'https://rubygems.org'
-gemspec
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -1,67 +1,67 @@
-# Logstash input plugin for Azure diagnostics data from Storage Tables 
-
-## Summary
-This plugin reads Azure diagnostics data from specified Azure Storage Table and parses the data for output.
-
-## Installation
-You can install this plugin using the Logstash "plugin" or "logstash-plugin" (for newer versions of Logstash) command:
-```sh
-logstash-plugin install logstash-input-azurewadtable
-```
-For more information, see Logstash reference [Working with plugins](https://www.elastic.co/guide/en/logstash/current/working-with-plugins.html).
-
-## Configuration
-### Required Parameters
-__*account_name*__
-
-The Azure Storage account name.
-
-__*access_key*__
-
-The access key to the storage account.
-
-__*table_name*__
-
-The storage table to pull data from.
-
-### Optional Parameters
-__*entity_count_to_process*__
-
-The plugin queries and processes table entities in a loop, this parameter is to specify the maximum number of entities it should query and process per loop. The default value is 100.
-
-__*collection_start_time_utc*__
-
-Specifies the point of time after which the entities created should be included in the query results. The default value is when the plugin gets initialized:
-
-```ruby
-Time.now.utc.iso8601
-```
-__*etw_pretty_print*__
-
-True to pretty print ETW files, otherwise False. The default value is False.
-
-__*idle_delay_seconds*__
-
-Specifies the seconds to wait between each processing loop. The default value is 15.  
-
-__*endpoint*__
-
-Specifies the endpoint of Azure environment. The default value is "core.windows.net".  
-
-### Examples
-```
-input
-{
-    azurewadeventhub
-    {
-        account_name => "mystorageaccount"
-        access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
-        table_name => "WADWindowsEventLogsTable"
-    }
-}
-```
-
-## More information
-The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.
-
-Please also see [Analyze Diagnostics Data with ELK template](https://github.com/Azure/azure-quickstart-templates/tree/master/diagnostics-with-elk) for quick deployment of ELK to Azure.   
+# Logstash input plugin for Azure diagnostics data from Storage Tables 
+
+## Summary
+This plugin reads Azure diagnostics data from specified Azure Storage Table and parses the data for output.
+
+## Installation
+You can install this plugin using the Logstash "plugin" or "logstash-plugin" (for newer versions of Logstash) command:
+```sh
+logstash-plugin install logstash-input-azurewadtable
+```
+For more information, see Logstash reference [Working with plugins](https://www.elastic.co/guide/en/logstash/current/working-with-plugins.html).
+
+## Configuration
+### Required Parameters
+__*account_name*__
+
+The Azure Storage account name.
+
+__*access_key*__
+
+The access key to the storage account.
+
+__*table_name*__
+
+The storage table to pull data from.
+
+### Optional Parameters
+__*entity_count_to_process*__
+
+The plugin queries and processes table entities in a loop, this parameter is to specify the maximum number of entities it should query and process per loop. The default value is 100.
+
+__*collection_start_time_utc*__
+
+Specifies the point of time after which the entities created should be included in the query results. The default value is when the plugin gets initialized:
+
+```ruby
+Time.now.utc.iso8601
+```
+__*etw_pretty_print*__
+
+True to pretty print ETW files, otherwise False. The default value is False.
+
+__*idle_delay_seconds*__
+
+Specifies the seconds to wait between each processing loop. The default value is 15.  
+
+__*endpoint*__
+
+Specifies the endpoint of Azure environment. The default value is "core.windows.net".  
+
+### Examples
+```
+input
+{
+    azurewadtable
+    {
+        account_name => "mystorageaccount"
+        access_key => "VGhpcyBpcyBhIGZha2Uga2V5Lg=="
+        table_name => "WADWindowsEventLogsTable"
+    }
+}
+```
+
+## More information
+The source code of this plugin is hosted in GitHub repo [Microsoft Azure Diagnostics with ELK](https://github.com/Azure/azure-diagnostics-tools). We welcome you to provide feedback and/or contribute to the project.
+
+Please also see [Analyze Diagnostics Data with ELK template](https://github.com/Azure/azure-quickstart-templates/tree/master/diagnostics-with-elk) for quick deployment of ELK to Azure.   

data/lib/logstash/inputs/azurewadtable.rb

@@ -1,134 +1,174 @@
-# encoding: utf-8
-require "logstash/inputs/base"
-require "logstash/namespace"
-
-require "azure"
-
-class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
-  class Interrupted < StandardError; end
-
-  config_name "azurewadtable"
-  milestone 1
-  
-  config :account_name, :validate => :string
-  config :access_key, :validate => :string
-  config :table_name, :validate => :string
-  config :entity_count_to_process, :validate => :string, :default => 100
-  config :collection_start_time_utc, :validate => :string, :default => Time.now.utc.iso8601
-  config :etw_pretty_print, :validate => :boolean, :default => false
-  config :idle_delay_seconds, :validate => :number, :default => 15
-  config :endpoint, :validate => :string, :default => "core.windows.net"
-
-  TICKS_SINCE_EPOCH = Time.utc(0001, 01, 01).to_i * 10000000
-
-  def initialize(*args)
-    super(*args)
-  end # initialize
-
-  public
-  def register
-    Azure.configure do |config|
-      config.storage_account_name = @account_name
-      config.storage_access_key = @access_key
-      config.storage_table_host = "https://#{@account_name}.table.#{@endpoint}"
-     end
-    @azure_table_service = Azure::Table::TableService.new
-    @last_timestamp = @collection_start_time_utc
-    @idle_delay = @idle_delay_seconds
-    @continuation_token = nil
-  end # register
-  
-  public
-  def run(output_queue)
-    while !stop?
-      @logger.debug("Starting process method @" + Time.now.to_s);
-      process(output_queue)
-      @logger.debug("Starting delay of: " + @idle_delay_seconds.to_s + " seconds @" + Time.now.to_s);
-      sleep @idle_delay
-    end # while
-  end # run
- 
-  public
-  def teardown
-  end  
-
-  def process(output_queue)
-    @logger.debug(@last_timestamp)
-    # query data using start_from_time
-    query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}')"
-    for i in 0..99
-      query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___9999999999999999999')"
-    end # for block
-    query_filter = query_filter.gsub('"','')
-    @logger.debug("Query filter: " + query_filter)
-    query = { :top => @entity_count_to_process, :filter => query_filter, :continuation_token => @continuation_token }
-    result = @azure_table_service.query_entities(@table_name, query)
-    @continuation_token = result.continuation_token
-
-    if result and result.length > 0
-      result.each do |entity|
-        event = LogStash::Event.new(entity.properties)
-        event["type"] = @table_name
-
-        # Help pretty print etw files
-        if (@etw_pretty_print && !event["EventMessage"].nil? && !event["Message"].nil?)
-          logger.debug("event: " + event.to_s)
-          eventMessage = event["EventMessage"].to_s
-          message = event["Message"].to_s
-          logger.debug("EventMessage: " + eventMessage)
-          logger.debug("Message: " + message)
-          if (eventMessage.include? "%")
-            logger.debug("starting pretty print")
-            toReplace = eventMessage.scan(/%\d+/)
-            payload = message.scan(/(?<!\\S)([a-zA-Z]+)=(\"[^\"]*\")(?!\\S)/)
-            # Split up the format string to seperate all of the numbers
-            toReplace.each do |key| 
-              logger.debug("Replacing key: " + key.to_s)
-              index = key.scan(/\d+/).join.to_i
-              newValue = payload[index - 1][1]
-              logger.debug("New Value: " + newValue)
-              eventMessage[key] = newValue
-            end # do block
-            event["EventMessage"] = eventMessage
-            logger.debug("pretty print end. result: " + event["EventMessage"].to_s)
-          end
-        end
-
-        output_queue << event
-      end # each block
-      @idle_delay = 0
-      @last_timestamp = result.last.properties["TIMESTAMP"].iso8601 unless @continuation_token
-    else
-      @logger.debug("No new results found.")
-      @idle_delay = @idle_delay_seconds
-    end # if block
-    
-  rescue => e
-    @logger.error("Oh My, An error occurred.", :exception => e)
-    raise
-  end # process
-
-  # Windows Azure Diagnostic's algorithm for determining the partition key based on time is as follows:
-  # 1. Take time in UTC without seconds.
-  # 2. Convert it into .net ticks
-  # 3. add a '0' prefix.
-  def partitionkey_from_datetime(time_string)
-    collection_time = Time.parse(time_string)
-    if collection_time
-      @logger.debug("collection time parsed successfully #{collection_time}")
-    else
-      raise(ArgumentError, "Could not parse the time_string")
-    end # if else block
-
-    collection_time -= collection_time.sec
-    ticks = to_ticks(collection_time)
-    "0#{ticks}"
-  end # partitionkey_from_datetime
-  
-  # Convert time to ticks
-  def to_ticks(time_to_convert)
-    @logger.debug("Converting time to ticks")
-    time_to_convert.to_i * 10000000 - TICKS_SINCE_EPOCH 
-  end # to_ticks
-
-end # LogStash::Inputs::AzureWADTable
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "time"
+require "azure"
+
+class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+
+  config_name "azurewadtable"
+  milestone 1
+
+  config :account_name, :validate => :string
+  config :access_key, :validate => :string
+  config :table_name, :validate => :string
+  config :entity_count_to_process, :validate => :string, :default => 100
+  config :collection_start_time_utc, :validate => :string, :default => Time.now.utc.iso8601
+  config :etw_pretty_print, :validate => :boolean, :default => false
+  config :idle_delay_seconds, :validate => :number, :default => 15
+  config :endpoint, :validate => :string, :default => "core.windows.net"
+
+  # Default 1 minute delay to ensure all data is published to the table before querying.
+  # See issue #23 for more: https://github.com/Azure/azure-diagnostics-tools/issues/23
+  config :data_latency_minutes, :validate => :number, :default => 1
+
+  TICKS_SINCE_EPOCH = Time.utc(0001, 01, 01).to_i * 10000000
+
+  def initialize(*args)
+    super(*args)
+  end # initialize
+
+  public
+  def register
+    Azure.configure do |config|
+      config.storage_account_name = @account_name
+      config.storage_access_key = @access_key
+      config.storage_table_host = "https://#{@account_name}.table.#{@endpoint}"
+     end
+    @azure_table_service = Azure::Table::TableService.new
+    @last_timestamp = @collection_start_time_utc
+    @idle_delay = @idle_delay_seconds
+    @continuation_token = nil
+  end # register
+
+  public
+  def run(output_queue)
+    while !stop?
+      @logger.debug("Starting process method @" + Time.now.to_s);
+      process(output_queue)
+      @logger.debug("Starting delay of: " + @idle_delay_seconds.to_s + " seconds @" + Time.now.to_s);
+      sleep @idle_delay
+    end # while
+  end # run
+
+  public
+  def teardown
+  end
+
+  def build_latent_query
+    @logger.debug("from #{@last_timestamp} to #{@until_timestamp}")
+    query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{partitionkey_from_datetime(@until_timestamp)}')"
+    for i in 0..99
+      query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@until_timestamp)}')"
+    end # for block
+    query_filter = query_filter.gsub('"','')
+    query_filter
+  end
+
+  def build_zero_latency_query
+    @logger.debug("from #{@last_timestamp} to most recent data")
+    # query data using start_from_time
+    query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}')"
+    for i in 0..99
+      query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___9999999999999999999')"
+    end # for block
+    query_filter = query_filter.gsub('"','')
+    query_filter
+  end
+
+  def process(output_queue)
+    if @data_latency_minutes > 0
+      @until_timestamp = (Time.now - (60 * @data_latency_minutes)).iso8601 unless @continuation_token
+      query_filter = build_latent_query
+    else
+      query_filter = build_zero_latency_query
+    end
+    @logger.debug("Query filter: " + query_filter)
+    query = { :top => @entity_count_to_process, :filter => query_filter, :continuation_token => @continuation_token }
+    result = @azure_table_service.query_entities(@table_name, query)
+    @continuation_token = result.continuation_token
+
+    if result and result.length > 0
+      @logger.debug("#{result.length} results found.")
+      last_good_timestamp = nil
+      result.each do |entity|
+        event = LogStash::Event.new(entity.properties)
+        event.set("type", @table_name)
+
+        # Help pretty print etw files
+        if (@etw_pretty_print && !event.get("EventMessage").nil? && !event.get("Message").nil?)
+          @logger.debug("event: " + event.to_s)
+          eventMessage = event.get("EventMessage").to_s
+          message = event.get("Message").to_s
+          @logger.debug("EventMessage: " + eventMessage)
+          @logger.debug("Message: " + message)
+          if (eventMessage.include? "%")
+            @logger.debug("starting pretty print")
+            toReplace = eventMessage.scan(/%\d+/)
+            payload = message.scan(/(?<!\\S)([a-zA-Z]+)=(\"[^\"]*\")(?!\\S)/)
+            # Split up the format string to seperate all of the numbers
+            toReplace.each do |key|
+              @logger.debug("Replacing key: " + key.to_s)
+              index = key.scan(/\d+/).join.to_i
+              newValue = payload[index - 1][1]
+              @logger.debug("New Value: " + newValue)
+              eventMessage[key] = newValue
+            end # do block
+            event.set("EventMessage", eventMessage)
+            @logger.debug("pretty print end. result: " + event.get("EventMessage").to_s)
+          end
+        end
+        decorate(event)
+        if event.get('PreciseTimeStamp').is_a?(Time)
+          event.set('PreciseTimeStamp', LogStash::Timestamp.new(event.get('PreciseTimeStamp')))
+        end
+        theTIMESTAMP = event.get('TIMESTAMP')
+        if theTIMESTAMP.is_a?(LogStash::Timestamp)
+          last_good_timestamp = theTIMESTAMP.to_iso8601
+        elsif theTIMESTAMP.is_a?(Time)
+          last_good_timestamp = theTIMESTAMP.iso8601
+          event.set('TIMESTAMP', LogStash::Timestamp.new(theTIMESTAMP))
+        else
+          @logger.warn("Found result with invalid TIMESTAMP. " + event.to_hash.to_s)
+        end
+        output_queue << event
+      end # each block
+      @idle_delay = 0
+      if (!last_good_timestamp.nil?)
+        @last_timestamp = last_good_timestamp unless @continuation_token
+      end
+    else
+      @logger.debug("No new results found.")
+      @idle_delay = @idle_delay_seconds
+    end # if block
+
+  rescue => e
+    @logger.error("Oh My, An error occurred.", :exception => e)
+    raise
+  end # process
+
+  # Windows Azure Diagnostic's algorithm for determining the partition key based on time is as follows:
+  # 1. Take time in UTC without seconds.
+  # 2. Convert it into .net ticks
+  # 3. add a '0' prefix.
+  def partitionkey_from_datetime(time_string)
+    collection_time = Time.parse(time_string)
+    if collection_time
+      @logger.debug("collection time parsed successfully #{collection_time}")
+    else
+      raise(ArgumentError, "Could not parse the time_string")
+    end # if else block
+
+    collection_time -= collection_time.sec
+    ticks = to_ticks(collection_time)
+    "0#{ticks}"
+  end # partitionkey_from_datetime
+
+  # Convert time to ticks
+  def to_ticks(time_to_convert)
+    @logger.debug("Converting time to ticks")
+    time_to_convert.to_i * 10000000 - TICKS_SINCE_EPOCH
+  end # to_ticks
+
+end # LogStash::Inputs::AzureWADTable

data/logstash-input-azurewadtable.gemspec

@@ -1,25 +1,25 @@
-Gem::Specification.new do |s|
-  s.name            = 'logstash-input-azurewadtable'
-  s.version         = '0.9.9'
-  s.licenses        = ['Apache License (2.0)']
-  s.summary         = "This plugin collects Microsoft Azure Diagnostics data from Azure Storage Tables."
-  s.description     = "This gem is a Logstash plugin. It reads and parses diagnostics data from Azure Storage Tables."
-  s.authors         = ["Microsoft Corporation"]
-  s.email           = 'azdiag@microsoft.com'
-  s.homepage        = "https://github.com/Azure/azure-diagnostics-tools"
-  s.require_paths   = ["lib"]
-
-  # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
-   # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
-
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
-
-  # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', '~> 2.0'
-  s.add_runtime_dependency 'azure', '~> 0.7.3'
-  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
-end
-
+Gem::Specification.new do |s|
+  s.name            = 'logstash-input-azurewadtable'
+  s.version         = '0.9.11'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "This plugin collects Microsoft Azure Diagnostics data from Azure Storage Tables."
+  s.description     = "This gem is a Logstash plugin. It reads and parses diagnostics data from Azure Storage Tables."
+  s.authors         = ["Microsoft Corporation"]
+  s.email           = 'azdiag@microsoft.com'
+  s.homepage        = "https://github.com/Azure/azure-diagnostics-tools"
+  s.require_paths   = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'azure', '~> 0.7.3'
+  s.add_development_dependency 'logstash-devutils', '>= 1.1.0'
+end
+

metadata

@@ -1,57 +1,63 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azurewadtable
 version: !ruby/object:Gem::Version
-  version: 0.9.9
+  version: 0.9.11
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-27 00:00:00.000000000 Z
+date: 2016-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
-  name: logstash-core
+        version: '2.99'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: azure
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
+        version: 0.7.3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.3
-  name: azure
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.3
-- !ruby/object:Gem::Dependency
+        version: 1.1.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.0.16
-  name: logstash-devutils
+        version: 1.1.0
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.16
 description: This gem is a Logstash plugin. It reads and parses diagnostics data from Azure Storage Tables.
 email: azdiag@microsoft.com
 executables: []
@@ -77,12 +83,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []