logstash-input-azurewadtable 0.9.11 → 0.9.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c27925f10618e725258c57b9afb50b9f8c25689c
-  data.tar.gz: 3f4a6dc33cbf979edeaf97b6696470b22bd7f5ff
+SHA256:
+  metadata.gz: 5d5f03c67ab4e356e775d2c01ea283c5ab984ed9fe33cbf588406392fa1b1422
+  data.tar.gz: 86c7c36b5735e0119e39a1764137736b2fc1cd824126f33696073fbec48ffc6d
 SHA512:
-  metadata.gz: 11ac6192c5c76e265bab0412520b795e898e28e5861aeacdebdfd05634da3c718bc4b4aa079f4706959fe63c4ddca2191ae21505785067516934a1028153947c
-  data.tar.gz: 1581ff53ae508173137aa0ccb8ad96fc5c224f4385bd6b7f1535362196d4c44c050eef4aa4b676d3df162141941b85ff2ec94aca18a62626fa31149f3087be2b
+  metadata.gz: 1e8ce77d95b549f2da0eb56ee937604b11bf49ecb6b4b68751246d1c44c03c47d0f88baa9e7e2f263b6acfff8db22f00ee7c0012d20f66a717a54fe6b796f9c3
+  data.tar.gz: 6478098b9b5028439215ec1f1c2386eecfe9688ab931590660a439f80584b6dc84321242a2a355f862b0407bdbbb8f70f2477ef7ee57cf3eaf69a6c0bf82c1a0

data/README.md

@@ -48,6 +48,9 @@ __*endpoint*__
 
 Specifies the endpoint of Azure environment. The default value is "core.windows.net".  
 
+__*past_queries_count*__
+Specifies the number of past queries to run so the plugin doesn't miss late arriving data. By default this is 5
+
 ### Examples
 ```
 input

data/lib/logstash/inputs/azurewadtable.rb

@@ -2,7 +2,8 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "time"
-require "azure"
+require "azure/storage"
+require "set"
 
 class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
   class Interrupted < StandardError; end
@@ -14,7 +15,7 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
   config :access_key, :validate => :string
   config :table_name, :validate => :string
   config :entity_count_to_process, :validate => :string, :default => 100
-  config :collection_start_time_utc, :validate => :string, :default => Time.now.utc.iso8601
+  config :collection_start_time_utc, :validate => :string, :default => nil #the actual value is set in the ctor (now - data_latency_minutes - 1)
   config :etw_pretty_print, :validate => :boolean, :default => false
   config :idle_delay_seconds, :validate => :number, :default => 15
   config :endpoint, :validate => :string, :default => "core.windows.net"
@@ -23,23 +24,37 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
   # See issue #23 for more: https://github.com/Azure/azure-diagnostics-tools/issues/23
   config :data_latency_minutes, :validate => :number, :default => 1
 
+  # Number of past queries to be run, so we don't miss late arriving data
+  config :past_queries_count, :validate => :number, :default => 5
+
   TICKS_SINCE_EPOCH = Time.utc(0001, 01, 01).to_i * 10000000
 
+  INITIAL_QUERY_SPLIT_PERIOD_MINUTES = 30
+
   def initialize(*args)
     super(*args)
+    if @collection_start_time_utc.nil?
+      @collection_start_time_utc = (Time.now - ( 60 * @data_latency_minutes) - 60).iso8601
+      @logger.debug("collection_start_time_utc = #{@collection_start_time_utc}")
+    end
   end # initialize
 
   public
   def register
-    Azure.configure do |config|
-      config.storage_account_name = @account_name
-      config.storage_access_key = @access_key
-      config.storage_table_host = "https://#{@account_name}.table.#{@endpoint}"
-     end
-    @azure_table_service = Azure::Table::TableService.new
+    user_agent = "logstash-input-azurewadtable"
+    user_agent << "/" << Gem.latest_spec_for("logstash-input-azurewadtable").version.to_s
+
+    @client = Azure::Storage::Client.create(
+      :storage_account_name => @account_name,
+      :storage_access_key => @access_key,
+      :storage_table_host => "https://#{@account_name}.table.#{@endpoint}",
+      :user_agent_prefix => user_agent)
+    @azure_table_service = @client.table_client
+
     @last_timestamp = @collection_start_time_utc
     @idle_delay = @idle_delay_seconds
-    @continuation_token = nil
+    @duplicate_detector = DuplicateDetector.new(@logger, @past_queries_count)
+    @first_run = true
   end # register
 
   public
@@ -47,7 +62,7 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
     while !stop?
       @logger.debug("Starting process method @" + Time.now.to_s);
       process(output_queue)
-      @logger.debug("Starting delay of: " + @idle_delay_seconds.to_s + " seconds @" + Time.now.to_s);
+      @logger.debug("Starting delay of: " + @idle_delay.to_s + " seconds @" + Time.now.to_s);
       sleep @idle_delay
     end # while
   end # run
@@ -58,96 +73,105 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
 
   def build_latent_query
     @logger.debug("from #{@last_timestamp} to #{@until_timestamp}")
+    if @last_timestamp > @until_timestamp
+      @logger.debug("last_timestamp is in the future. Will not run any query!")
+      return nil
+    end
     query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{partitionkey_from_datetime(@until_timestamp)}')"
     for i in 0..99
       query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@until_timestamp)}')"
     end # for block
     query_filter = query_filter.gsub('"','')
-    query_filter
-  end
-
-  def build_zero_latency_query
-    @logger.debug("from #{@last_timestamp} to most recent data")
-    # query data using start_from_time
-    query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}')"
-    for i in 0..99
-      query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___9999999999999999999')"
-    end # for block
-    query_filter = query_filter.gsub('"','')
-    query_filter
+    return AzureQuery.new(@logger, @azure_table_service, @table_name, query_filter, @last_timestamp.to_s + "-" + @until_timestamp.to_s, @entity_count_to_process)
   end
 
   def process(output_queue)
-    if @data_latency_minutes > 0
-      @until_timestamp = (Time.now - (60 * @data_latency_minutes)).iso8601 unless @continuation_token
-      query_filter = build_latent_query
-    else
-      query_filter = build_zero_latency_query
+    @until_timestamp = (Time.now - (60 * @data_latency_minutes)).iso8601
+    last_good_timestamp = nil
+
+     # split first query so we don't fetch old data several times for no reason
+    if @first_run
+      @first_run = false
+      diff = DateTime.iso8601(@until_timestamp).to_time - DateTime.iso8601(@last_timestamp).to_time
+      if diff > INITIAL_QUERY_SPLIT_PERIOD_MINUTES * 60
+        @logger.debug("Splitting initial query in two")
+        original_until = @until_timestamp
+
+        @until_timestamp = (DateTime.iso8601(@until_timestamp).to_time - INITIAL_QUERY_SPLIT_PERIOD_MINUTES * 60).iso8601
+
+        query = build_latent_query
+        @duplicate_detector.filter_duplicates(query, ->(entity) {
+          on_new_data(entity, output_queue, last_good_timestamp)
+        }, false)
+
+        @last_timestamp = (DateTime.iso8601(@until_timestamp).to_time - 1).iso8601
+        @until_timestamp = original_until
+      end
     end
-    @logger.debug("Query filter: " + query_filter)
-    query = { :top => @entity_count_to_process, :filter => query_filter, :continuation_token => @continuation_token }
-    result = @azure_table_service.query_entities(@table_name, query)
-    @continuation_token = result.continuation_token
-
-    if result and result.length > 0
-      @logger.debug("#{result.length} results found.")
-      last_good_timestamp = nil
-      result.each do |entity|
-        event = LogStash::Event.new(entity.properties)
-        event.set("type", @table_name)
-
-        # Help pretty print etw files
-        if (@etw_pretty_print && !event.get("EventMessage").nil? && !event.get("Message").nil?)
-          @logger.debug("event: " + event.to_s)
-          eventMessage = event.get("EventMessage").to_s
-          message = event.get("Message").to_s
-          @logger.debug("EventMessage: " + eventMessage)
-          @logger.debug("Message: " + message)
-          if (eventMessage.include? "%")
-            @logger.debug("starting pretty print")
-            toReplace = eventMessage.scan(/%\d+/)
-            payload = message.scan(/(?<!\\S)([a-zA-Z]+)=(\"[^\"]*\")(?!\\S)/)
-            # Split up the format string to seperate all of the numbers
-            toReplace.each do |key|
-              @logger.debug("Replacing key: " + key.to_s)
-              index = key.scan(/\d+/).join.to_i
-              newValue = payload[index - 1][1]
-              @logger.debug("New Value: " + newValue)
-              eventMessage[key] = newValue
-            end # do block
-            event.set("EventMessage", eventMessage)
-            @logger.debug("pretty print end. result: " + event.get("EventMessage").to_s)
-          end
-        end
-        decorate(event)
-        if event.get('PreciseTimeStamp').is_a?(Time)
-          event.set('PreciseTimeStamp', LogStash::Timestamp.new(event.get('PreciseTimeStamp')))
-        end
-        theTIMESTAMP = event.get('TIMESTAMP')
-        if theTIMESTAMP.is_a?(LogStash::Timestamp)
-          last_good_timestamp = theTIMESTAMP.to_iso8601
-        elsif theTIMESTAMP.is_a?(Time)
-          last_good_timestamp = theTIMESTAMP.iso8601
-          event.set('TIMESTAMP', LogStash::Timestamp.new(theTIMESTAMP))
-        else
-          @logger.warn("Found result with invalid TIMESTAMP. " + event.to_hash.to_s)
-        end
-        output_queue << event
-      end # each block
-      @idle_delay = 0
+
+    query = build_latent_query
+    filter_result = @duplicate_detector.filter_duplicates(query, ->(entity) {
+      last_good_timestamp = on_new_data(entity, output_queue, last_good_timestamp)
+    })
+
+    if filter_result
       if (!last_good_timestamp.nil?)
-        @last_timestamp = last_good_timestamp unless @continuation_token
+        @last_timestamp = last_good_timestamp
       end
     else
       @logger.debug("No new results found.")
-      @idle_delay = @idle_delay_seconds
-    end # if block
+    end
 
   rescue => e
-    @logger.error("Oh My, An error occurred.", :exception => e)
+    @logger.error("Oh My, An error occurred. Error:#{e}: Trace: #{e.backtrace}", :exception => e)
     raise
   end # process
 
+  def on_new_data(entity, output_queue, last_good_timestamp)
+    #@logger.debug("new event")
+    event = LogStash::Event.new(entity.properties)
+    event.set("type", @table_name)
+
+    # Help pretty print etw files
+    if (@etw_pretty_print && !event.get("EventMessage").nil? && !event.get("Message").nil?)
+      @logger.debug("event: " + event.to_s)
+      eventMessage = event.get("EventMessage").to_s
+      message = event.get("Message").to_s
+      @logger.debug("EventMessage: " + eventMessage)
+      @logger.debug("Message: " + message)
+      if (eventMessage.include? "%")
+        @logger.debug("starting pretty print")
+        toReplace = eventMessage.scan(/%\d+/)
+        payload = message.scan(/(?<!\\S)([a-zA-Z]+)=(\"[^\"]*\")(?!\\S)/)
+        # Split up the format string to seperate all of the numbers
+        toReplace.each do |key|
+          @logger.debug("Replacing key: " + key.to_s)
+          index = key.scan(/\d+/).join.to_i
+          newValue = payload[index - 1][1]
+          @logger.debug("New Value: " + newValue)
+          eventMessage[key] = newValue
+        end # do block
+        event.set("EventMessage", eventMessage)
+        @logger.debug("pretty print end. result: " + event.get("EventMessage").to_s)
+      end
+    end
+    decorate(event)
+    if event.get('PreciseTimeStamp').is_a?(Time)
+      event.set('PreciseTimeStamp', LogStash::Timestamp.new(event.get('PreciseTimeStamp')))
+    end
+    theTIMESTAMP = event.get('TIMESTAMP')
+    if theTIMESTAMP.is_a?(LogStash::Timestamp)
+      last_good_timestamp = theTIMESTAMP.to_iso8601
+    elsif theTIMESTAMP.is_a?(Time)
+      last_good_timestamp = theTIMESTAMP.iso8601
+      event.set('TIMESTAMP', LogStash::Timestamp.new(theTIMESTAMP))
+    else
+      @logger.warn("Found result with invalid TIMESTAMP. " + event.to_hash.to_s)
+    end
+    output_queue << event
+    return last_good_timestamp
+  end
+
   # Windows Azure Diagnostic's algorithm for determining the partition key based on time is as follows:
   # 1. Take time in UTC without seconds.
   # 2. Convert it into .net ticks
@@ -155,7 +179,7 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
   def partitionkey_from_datetime(time_string)
     collection_time = Time.parse(time_string)
     if collection_time
-      @logger.debug("collection time parsed successfully #{collection_time}")
+      #@logger.debug("collection time parsed successfully #{collection_time}")
     else
       raise(ArgumentError, "Could not parse the time_string")
     end # if else block
@@ -167,8 +191,154 @@ class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
 
   # Convert time to ticks
   def to_ticks(time_to_convert)
-    @logger.debug("Converting time to ticks")
+    #@logger.debug("Converting time to ticks")
     time_to_convert.to_i * 10000000 - TICKS_SINCE_EPOCH
   end # to_ticks
 
 end # LogStash::Inputs::AzureWADTable
+
+class AzureQuery
+  def initialize(logger, azure_table_service, table_name, query_str, query_id, entity_count_to_process)
+    @logger = logger
+    @query_str = query_str
+    @query_id = query_id
+    @entity_count_to_process = entity_count_to_process
+    @azure_table_service = azure_table_service
+    @table_name = table_name
+    @continuation_token = nil
+  end
+
+  def reset
+    @continuation_token = nil
+  end
+
+  def id
+    return @query_id
+  end
+
+  def run(on_result_cbk)
+    results_found = false
+    @logger.debug("[#{@query_id}]Query filter: " + @query_str)
+    begin
+      @logger.debug("[#{@query_id}]Running query. continuation_token: #{@continuation_token}")
+      query = { :top => @entity_count_to_process, :filter => @query_str, :continuation_token => @continuation_token }
+      result = @azure_table_service.query_entities(@table_name, query)
+
+      if result and result.length > 0
+        results_found = true
+        @logger.debug("[#{@query_id}] #{result.length} results found.")
+        result.each do |entity|
+          on_result_cbk.call(entity)
+        end
+      end
+
+      @continuation_token = result.continuation_token
+    end until !@continuation_token
+
+    return results_found
+  end
+end
+
+class QueryData
+  def initialize(logger, query)
+    @logger = logger
+    @query = query
+    @results_cache = Set.new
+  end
+
+  def id
+    return @query.id
+  end
+
+  def get_unique_id(entity)
+    uniqueId = ""
+    partitionKey = entity.properties["PartitionKey"]
+    rowKey = entity.properties["RowKey"]
+    uniqueId << partitionKey << "#" << rowKey
+    return uniqueId
+  end
+
+  def run_query(on_new_entity_cbk)
+    @query.reset
+    @query.run( ->(entity) {
+      uniqueId = get_unique_id(entity)
+
+      if @results_cache.add?(uniqueId).nil?
+        @logger.debug("[#{@query.id}][QueryData] #{uniqueId} already processed")
+      else
+        @logger.debug("[#{@query.id}][QueryData] #{uniqueId} new item")
+        on_new_entity_cbk.call(entity)
+      end
+    })
+  end
+
+  def has_entity(entity)
+    return @results_cache.include?(get_unique_id(entity))
+  end
+
+end
+
+class DuplicateDetector
+  def initialize(logger, past_queries_count)
+    @logger = logger
+    @past_queries_count = past_queries_count
+    @query_cache = []
+  end
+
+  def filter_duplicates(query, on_new_item_ckb, should_cache_query = true)
+    if query.nil?
+      @logger.debug("query is nil")
+      return false
+    end
+    #push in front, pop from the back
+    latest_query = QueryData.new(@logger, query)
+    @query_cache.insert(0, latest_query)
+
+    found_new_items = false
+
+    # results is most likely empty or has very few items for older queries (most or all should be de-duplicated by run_query)
+    index = 0
+    @query_cache.each do |query_data|
+        query_data.run_query(->(entity) {
+        unique_id = query_data.get_unique_id(entity)
+
+        # queries overlap. Check for duplicates in all results
+        is_duplicate = false
+        for j in 0..@query_cache.length - 1
+          if j == index
+            next
+          end
+          q = @query_cache[j]
+          if q.has_entity(entity)
+            @logger.debug("[#{query_data.id}][filter_duplicates] #{unique_id} was already processed by #{q.id}")
+            is_duplicate = true
+            break
+          end
+        end
+
+        if !is_duplicate
+          found_new_items = true
+          @logger.debug("[#{query_data.id}][filter_duplicates] #{unique_id} new item")
+          on_new_item_ckb.call(entity)
+        end
+
+      })
+
+      index+=1
+    end
+
+    if !should_cache_query
+      @logger.debug("Removing first item from queue")
+      @query_cache.shift
+    end
+
+    @logger.debug("Query Cache length: #{@query_cache.length}")
+    until @query_cache.length <= @past_queries_count do
+      @query_cache.pop
+      @logger.debug("New Query Cache length: #{@query_cache.length}")
+    end
+
+    return found_new_items
+  end
+
+end

data/logstash-input-azurewadtable.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-azurewadtable'
-  s.version         = '0.9.11'
+  s.version         = '0.9.12'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This plugin collects Microsoft Azure Diagnostics data from Azure Storage Tables."
   s.description     = "This gem is a Logstash plugin. It reads and parses diagnostics data from Azure Storage Tables."
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency 'azure', '~> 0.7.3'
+  s.add_runtime_dependency 'azure-storage', '~> 0.13.0.preview'
   s.add_development_dependency 'logstash-devutils', '>= 1.1.0'
 end
 

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-azurewadtable
 version: !ruby/object:Gem::Version
-  version: 0.9.11
+  version: 0.9.12
 platform: ruby
 authors:
 - Microsoft Corporation
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-08 00:00:00.000000000 Z
+date: 2017-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -20,7 +19,10 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
@@ -28,37 +30,36 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: azure
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.3
+        version: 0.13.0.preview
+  name: azure-storage
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: 0.13.0.preview
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.0
+  name: logstash-devutils
   prerelease: false
   type: :development
-description: This gem is a Logstash plugin. It reads and parses diagnostics data from Azure Storage Tables.
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+description: This gem is a Logstash plugin. It reads and parses diagnostics data from
+  Azure Storage Tables.
 email: azdiag@microsoft.com
 executables: []
 extensions: []
@@ -93,9 +94,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
-summary: This plugin collects Microsoft Azure Diagnostics data from Azure Storage Tables.
+summary: This plugin collects Microsoft Azure Diagnostics data from Azure Storage
+  Tables.
 test_files:
 - spec/inputs/azurewadtable_spec.rb