generik-logstash-input-azureblob 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f5020155403abd1393bc355a70f783a0674f8886
+  data.tar.gz: f4df1a5e97fcdd9217653639f162ae2329c9df4f
+SHA512:
+  metadata.gz: b6ae69be71131a5798eb142a1e262d39df13faa9183a78263716fa8653f0b1b094e0dfac7cfc44390a241ba5c3883031110c4a823e24704cf69a69399a972be8
+  data.tar.gz: 88e0cd67a2d08a24b47f98641e88e25885ce753286f6fe519fccdee74547a4d077b45ea136b97de0d6c399a225356ddfc328525351c0a8caa078f529a55e2d59

data/lib/logstash/codecs/json_list.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+require "logstash/codecs/base"
+require "logstash/codecs/line"
+require "json"
+
+class LogStash::Codecs::JSONList < LogStash::Codecs::Base
+  config_name "json_list"
+  
+  milestone 0
+  
+  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
+
+  public
+  def register
+  end # def register
+  
+  public
+  def decode(data)
+    begin
+      JSON.parse(data).each do |obj|
+        yield LogStash::Event.new(obj)
+      end
+    rescue JSON::ParserError => e
+      @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
+      yield LogStash::Event.new("message" => event["message"])
+    end   
+  end # def decode
+  
+  public
+  def encode(data)
+    arr = Array.new
+    arr << data
+    @on_event.call(arr.to_json)
+  end # def encode
+
+end # class LogStash::Codecs::JSONList

data/lib/logstash/inputs/azureblob.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "azure"
+require "securerandom"
+
+# Reads events from Azure Blobs
+class LogStash::Inputs::Azureblob < LogStash::Inputs::Base
+
+  config_name "azureblob"
+  milestone 0
+  
+  default :codec, "json_lines"
+  
+  config :storage_account_name, :validate => :string
+  config :storage_access_key, :validate => :string
+  
+  config :container, :validate => :string
+  config :sleep_time, :validate => :number, :default => 10
+  
+  def initialize(*args)
+    super(*args)
+  end # def initialize
+  
+  public
+  def register
+    Azure.configure do |config|
+      config.storage_account_name = @storage_account_name
+      config.storage_access_key = @storage_access_key
+    end
+    @azure_blob = Azure::Blob::BlobService.new
+  end # def register
+  
+  def list_blob_names
+    blob_names = Set.new []
+    loop do
+      continuation_token = NIL
+      entries = @azure_blob.list_blobs(@container, { :timeout => 10, :marker => continuation_token})
+      entries.each do |entry|
+        blob_names << entry.name
+      end
+      continuation_token = entries.continuation_token
+      break if continuation_token.empty?
+    end
+    return blob_names
+  end # def list_blobs
+  
+  def acquire_lock(blob_name)
+    @azure_blob.create_page_blob(@container, blob_name, 512)
+    @azure_blob.acquire_lease(@container, blob_name,{:duration=>60, :timeout=>10, :proposed_lease_id=>SecureRandom.uuid})
+    return true
+  rescue LogStash::ShutdownSignal => e
+    raise e
+  rescue => e
+    @logger.error("Caught exception while locking", :exception => e)
+    return false
+  end # def acquire_lock
+  
+  def lock_blob(blob_names)
+    real_blob_names = blob_names.select { |name| !name.end_with?(".lock") }
+    real_blob_names.each do |blob_name|
+      if !blob_names.include?(blob_name + ".lock")
+        if acquire_lock(blob_name + ".lock")
+          return blob_name
+        end
+      end
+    end
+    return NIL
+  end # def lock_blob
+  
+  def process(output_queue)
+    blob_names = list_blob_names
+    blob_name = lock_blob(blob_names)
+    return if !blob_name
+    blob, content = @azure_blob.get_blob(@container, blob_name)
+    @codec.decode(content) do |event|
+      output_queue << event
+    end
+  rescue LogStash::ShutdownSignal => e
+    raise e
+  rescue => e
+    @logger.error("Oh My, An error occurred.", :exception => e)
+  end # def process
+  
+  public
+  def run(output_queue)
+    while true
+      process(output_queue)
+    end # loop
+  end # def run
+ 
+  public
+  def teardown
+  end # def teardown
+end # class LogStash::Inputs::Azuretopic

data/lib/logstash/inputs/azuretopic.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "azure"
+
+# Reads events from Azure topics
+class LogStash::Inputs::Azuretopic < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+
+  config_name "azuretopic"
+  milestone 0
+  
+  default :codec, "json_list"
+  
+  config :namespace, :validate => :string
+  config :access_key, :validate => :string
+  config :subscription, :validate => :string
+  config :topic, :validate => :string
+  config :deliverycount, :validate => :number, :default => 10
+  
+  def initialize(*args)
+	super(*args)
+  end # def initialize
+  
+  public
+  def register
+	  Azure.configure do |config|
+	    config.sb_namespace = @namespace
+	    config.sb_access_key = @access_key
+ 	  end
+	@azure_service_bus = Azure::ServiceBus::ServiceBusService.new
+  end # def register
+  
+  def process(output_queue)
+    message = @azure_service_bus.receive_subscription_message(@topic ,@subscription, { :peek_lock => true, :timeout => 1 } )
+    if message
+	  codec.decode(message.body) do |event|
+        output_queue << event
+	  end # codec.decode
+	  @azure_service_bus.delete_subscription_message(message)
+	end
+  rescue LogStash::ShutdownSignal => e
+    raise e
+  rescue => e
+    @logger.error("Oh My, An error occurred.", :exception => e)
+	if message and message.delivery_count > @deliverycount
+		@azure_service_bus.delete_subscription_message(message)
+	end
+  end # def process
+  
+  public
+  def run(output_queue)
+	while true
+      process(output_queue)
+	end # loop
+  end # def run
+ 
+  public
+  def teardown
+  end # def teardown
+end # class LogStash::Inputs::Azuretopic

data/lib/logstash/inputs/azuretopicthreadable.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "thread"
+
+require "azure"
+
+# Reads events from Azure topics
+class LogStash::Inputs::Azuretopicthreadable < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+
+  config_name "azuretopicthreadable"
+  milestone 0
+  
+  default :codec, "json" # default json codec
+  
+  config :namespace, :validate => :string
+  config :access_key, :validate => :string
+  config :subscription, :validate => :string
+  config :topic, :validate => :string
+  config :deliverycount, :validate => :number, :default => 10
+  config :threads, :validate => :number, :default => 1
+  config :thread_sleep_time, :validate => :number, :default => 1.0/50.0
+  
+  def initialize(*args)
+	super(*args)
+  end # def initialize
+  
+  public
+  def register
+	  # Configure credentials
+	  Azure.configure do |config|
+	    config.sb_namespace = @namespace
+	    config.sb_access_key = @access_key
+ 	  end
+  end # def register
+  
+  def process(output_queue, pid)
+    # Get a new instance of a service
+  	azure_service_bus = Azure::ServiceBus::ServiceBusService.new
+	while true
+		begin
+	    # check if we have a message in the subscription
+		message = azure_service_bus.receive_subscription_message(@topic ,@subscription, { :peek_lock => true, :timeout => 1 } )
+		if message
+		    # decoding returns a yield
+			codec.decode(message.body) do |event|
+				output_queue << event
+			end # codec.decode
+			# delete the message after reading it
+			azure_service_bus.delete_subscription_message(message)
+		end
+		rescue LogStash::ShutdownSignal => e
+			raise e
+		rescue => e
+			@logger.error("Oh My, An error occurred. Thread id:" + pid.to_s, :exception => e)
+			if message and message.delivery_count > @deliverycount
+				azure_service_bus.delete_subscription_message(message)
+			end
+		end
+	sleep(@thread_sleep_time)
+	end
+  end # def process
+  
+  public
+  def run(output_queue)
+	threads = []
+    (0..(@threads-1)).each do |pid|
+      threads << Thread.new { process(output_queue, pid) }
+    end
+    threads.each { |thr| thr.join }
+  end # def run
+ 
+  public
+  def teardown
+  end # def teardown
+end # class LogStash::Inputs::Azuretopic

data/lib/logstash/inputs/azurewadtable.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+
+require "azure"
+
+class LogStash::Inputs::AzureWADTable < LogStash::Inputs::Base
+  class Interrupted < StandardError; end
+
+  config_name "azurewadtable"
+  milestone 0
+  
+  config :account_name, :validate => :string
+  config :access_key, :validate => :string
+  config :table_name, :validate => :string
+  config :entity_count_to_process, :validate => :string, :default => 100
+  config :collection_start_time_utc, :validate => :string, :default => Time.now.utc.iso8601
+  config :etw_pretty_print, :validate => :boolean, :default => false
+  config :idle_delay_seconds, :validate => :number, :default => 15
+
+  TICKS_SINCE_EPOCH = Time.utc(0001, 01, 01).to_i * 10000000
+
+  def initialize(*args)
+    super(*args)
+  end # initialize
+
+  public
+  def register
+    Azure.configure do |config|
+      config.storage_account_name = @account_name
+      config.storage_access_key = @access_key
+     end
+    @azure_table_service = Azure::Table::TableService.new
+    @last_timestamp = @collection_start_time_utc
+    @idle_delay = @idle_delay_seconds
+    @continuation_token = nil
+  end # register
+  
+  public
+  def run(output_queue)
+    loop do
+	  @logger.debug("Starting process method @" + Time.now.to_s);
+      process(output_queue)
+	  @logger.debug("Starting delay of: " + @idle_delay_seconds.to_s + " seconds @" + Time.now.to_s);
+	  sleep @idle_delay
+    end # loop
+  end # run
+ 
+  public
+  def teardown
+  end  
+
+  def process(output_queue)
+    @logger.debug(@last_timestamp)
+    # query data using start_from_time
+    query_filter = "(PartitionKey gt '#{partitionkey_from_datetime(@last_timestamp)}')"
+    for i in 0..99
+      query_filter << " or (PartitionKey gt '#{i.to_s.rjust(19, '0')}___#{partitionkey_from_datetime(@last_timestamp)}' and PartitionKey lt '#{i.to_s.rjust(19, '0')}___9999999999999999999')"
+    end # for block
+    query_filter = query_filter.gsub('"','')
+    @logger.debug("Query filter: " + query_filter)
+    query = { :top => @entity_count_to_process, :filter => query_filter, :continuation_token => @continuation_token }
+    result = @azure_table_service.query_entities(@table_name, query)
+    @continuation_token = result.continuation_token
+    
+    if result and result.length > 0
+      result.each do |entity|
+        event = LogStash::Event.new(entity.properties)
+        event["type"] = @table_name
+
+		# Help pretty print etw files
+		if (@etw_pretty_print && !event["EventMessage"].nil? && !event["Message"].nil?)
+		  logger.debug("event: " + event.to_s)
+		  eventMessage = event["EventMessage"].to_s
+		  message = event["Message"].to_s
+		  logger.debug("EventMessage: " + eventMessage)
+		  logger.debug("Message: " + message)
+		  if (eventMessage.include? "%")
+		    logger.debug("starting pretty print")
+		    toReplace = eventMessage.scan(/%\d+/)
+		    payload = message.scan(/(?<!\\S)([a-zA-Z]+)=(\"[^\"]*\")(?!\\S)/)
+		    # Split up the format string to seperate all of the numbers
+	        toReplace.each do |key| 
+		      logger.debug("Replacing key: " + key.to_s)
+		      index = key.scan(/\d+/).join.to_i
+			  newValue = payload[index - 1][1]
+			  logger.debug("New Value: " + newValue)
+		      eventMessage[key] = newValue
+		    end
+		  event["EventMessage"] = eventMessage
+		  logger.debug("pretty print end. result: " + event["EventMessage"].to_s)
+		  end
+		end
+		
+        output_queue << event
+      end # each block
+      @idle_delay = 0
+      @last_timestamp = result.last.properties["TIMESTAMP"].iso8601 unless @continuation_token
+    else
+      @logger.debug("No new results found.")
+	  @idle_delay = @idle_delay_seconds
+    end # if block
+    
+  rescue => e
+    @logger.error("Oh My, An error occurred.", :exception => e)
+    raise
+  end # process
+
+  # Windows Azure Diagnostic's algorithm for determining the partition key based on time is as follows:
+  # 1. Take time in UTC without seconds.
+  # 2. Convert it into .net ticks
+  # 3. add a '0' prefix.
+  def partitionkey_from_datetime(time_string)
+    collection_time = Time.parse(time_string)
+    if collection_time
+      @logger.debug("collection time parsed successfully #{collection_time}")
+    else
+      raise(ArgumentError, "Could not parse the time_string")
+    end # if else block
+
+    collection_time -= collection_time.sec
+    ticks = to_ticks(collection_time)
+    "0#{ticks}"
+  end # partitionkey_from_datetime
+  
+  # Convert time to ticks
+  def to_ticks(time_to_convert)
+    @logger.debug("Converting time to ticks")
+    time_to_convert.to_i * 10000000 - TICKS_SINCE_EPOCH 
+  end # to_ticks
+
+end # LogStash::Inputs::AzureWADTable

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: generik-logstash-input-azureblob
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- gongdo
+- Microsoft
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: azure
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin in order to use Microsoft Azure as an input
+  source.
+email: gongdo@live.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/logstash/codecs/json_list.rb
+- lib/logstash/inputs/azureblob.rb
+- lib/logstash/inputs/azuretopic.rb
+- lib/logstash/inputs/azuretopicthreadable.rb
+- lib/logstash/inputs/azurewadtable.rb
+homepage: ''
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Logstash input source from Azure.
+test_files: []