fluent-plugin-azure-queue 0.0.1.pre → 0.0.2.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b6a171e37ab51926fea05fb9a3af2b1732c2dc06
-  data.tar.gz: 20de54cb2df97d3a1f2f25d2e66a30a6b8ea79e6
+  metadata.gz: 9b5f7a60d891724ad0d4ac6e48558010990ffa61
+  data.tar.gz: e8981830e82a093f33f84ec7280b6bdad1dd06f9
 SHA512:
-  metadata.gz: ef3d713e38a5cee209a69930dfba747ac37239d1ae443e7473de000391b558bd9eecae4f29b6e111e3d891bb1fe91b17b0dbe458dbf9a2905e16e27939966bd5
-  data.tar.gz: aef6bd13e15495361dcc8414d3b17d8abf92c13ca87b98bd6e76c057d0f2e7be0d68fe2b8a750ccbcad6b1818a88fef959369f2f0069397b3128e90d94651ef6
+  metadata.gz: d0b20d70a0f4b5ac513ecd770d3ea6f66c8fbbc20dae47806f39735a1185fdb2f47f6ffff331f4a6e6802f2f340eeff2ad8cfe6d3fdabdd79b91ba619b2cb05a
+  data.tar.gz: e9b764484e9efee4a4025d0fa5c38765deefee2a8a222209351c208aa12e9b7add59d170ffb11f92c2a3673996028e96004859bb1fd779de31e52e9ff84ba122

data/Gemfile

@@ -1,3 +1,3 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gemspec

data/Gemfile.lock

@@ -1,14 +1,18 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-azure-queue (0.0.1.pre.pre)
+    fluent-plugin-azure-queue (0.0.1.pre)
+      avro (>= 1.8)
       azure-storage (>= 0.12.3.preview, < 0.13)
+      concurrent-ruby
       fluentd (>= 0.12.2, < 0.14)
       nokogiri
 
 GEM
-  remote: http://rubygems.org/
+  remote: https://rubygems.org/
   specs:
+    avro (1.8.2)
+      multi_json
     azure-core (0.1.10)
       faraday (~> 0.9)
       faraday_middleware (~> 0.10)
@@ -16,8 +20,9 @@ GEM
       azure-core (~> 0.1)
       faraday (~> 0.9)
       faraday_middleware (~> 0.10)
+    concurrent-ruby (1.0.5)
     cool.io (1.5.1)
-    faraday (0.13.1)
+    faraday (0.13.0)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
@@ -36,8 +41,9 @@ GEM
     json (2.1.0)
     mini_portile2 (2.1.0)
     msgpack (1.1.0)
+    multi_json (1.12.1)
     multipart-post (2.0.0)
-    nokogiri (1.7.2)
+    nokogiri (1.6.8.1)
       mini_portile2 (~> 2.1.0)
     power_assert (1.0.2)
     rake (12.0.0)

data/README.md

@@ -1,10 +1,14 @@
-# A  [Fluentd](http://github.com/fluent/fluentd) plugin to read from azure queues
+# A  [Fluentd](http://github.com/fluent/fluentd) plugin to read from azure queues and event hubs
+The azure queue input plugin performs at about 30 messages/second in my tests. If you need more throughput from event hubs,
+ I suggest using the event hub capture plugin.
 
 ## Dependencies
 
 fluentd v.12
 
-## Input: Configuration
+## azure_queue Input Plugin
+
+### Input: Configuration
 
     <source>
       @type azure_queue
@@ -14,8 +18,7 @@ fluentd v.12
       storage_access_key my_storage_access_key
       queue_name my_storage_queue
       fetch_interval 5     
-      batch_size 10 
-      lease_time 30
+      lease_duration 30
     </source>
 
 **tag (required)**
@@ -38,14 +41,78 @@ The storage queue name
 
 The the record key to put the message data into. Default 'message'
 
-**fetch_interval**
+**lease_duration**
+
+The time to lease the messages for. Default 300
+
+**max_fetch_threads**
+
+The maximum number of threads to fetch and delete queue messages with. Default 30
+
+## Integration with Azure Event Hub
+
+You can use an azure function to forward messages from event hubs to storage queues for easy ingestion by this gem. This is not recommended for high volumes, but should serve as a stop gap until a complete azure event hub gem is created.
+
+```c#
+using System;
+using Microsoft.WindowsAzure.Storage.Queue;
+
+public static void Run(string[] hubMessages, ICollector<string> outputQueue, TraceWriter log)
+{
+    foreach (string message in hubMessages)
+    {
+        int bytes = message.Length * sizeof(Char);
+        if (bytes < 64000)
+        {
+            outputQueue.Add(message);
+        }
+        else 
+        {
+            log.Warning($"Message is larger than 64k with {bytes} bytes. Dropping message");
+        }
+    }
+}
+```
+## azure_event_hub_capture Input Plugin
+This plugin is designed to work with blobs stored to a container via [Azure Event Hubs Capture](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview)
+
+### Input: Configuration
+
+    <source>
+      @type azure_event_hub_capture
+
+      tag event_hub_input
+      storage_account_name my_storage_account
+      storage_access_key my_storage_access_key
+      container_name my_capture_container
+      fetch_interval 30     
+      lease_duration 30
+    </source>
+
+**tag (required)**
 
-How long to wait between getting messages from the queue. Default 5
+The tag for the input
 
-**batch_size**
+**storage_account_name (required)**
+
+The storage account name
+
+**storage_access_key (required)**
+
+The storage account access key
+
+**container_name (required)**
+
+The capture container name
+
+**message_key**
+
+The the record key to put the message data into. Default 'message'
+
+**fetch_interval**
 
-The maximum number of messages to pull from the queue at once. Default 10. Max 32
+The time in seconds to sleep between fetching the blob list. Default 30
 
-**lease_time**
+**lease_duration**
 
-The time to lease the messages for. Default 30
+The time to lease the messages for. Default 60

data/VERSION

@@ -1 +1 @@
-0.0.1.pre
+0.0.2.pre

data/fluent-plugin-azure-queue.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |gem|
   gem.add_dependency "fluentd", [">= 0.12.2", "< 0.14"]
   gem.add_dependency "azure-storage", [">= 0.12.3.preview", "< 0.13"]
   gem.add_dependency "nokogiri"
+  gem.add_dependency "concurrent-ruby"
+  gem.add_dependency "avro", ">= 1.8"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "flexmock", ">= 1.3.3"

data/lib/fluent/plugin/in_azure_event_hub_capture.rb

@@ -0,0 +1,121 @@
+require 'fluent/input'
+require 'azure/storage'
+require "avro"
+
+module Fluent
+  class AzureEventHubCaptureInput < Input
+    Fluent::Plugin.register_input('azure_event_hub_capture', self)
+
+    desc 'Tag of the output events.'
+    config_param :tag, :string
+    desc 'The azure storage account name'
+    config_param :storage_account_name, :string
+    desc 'The azure storage account access key'
+    config_param :storage_access_key, :string
+    desc 'The container name'
+    config_param :container_name, :string
+    desc 'The the record key to put the message data into'
+    config_param :message_key, :string, default: 'message'
+    desc 'The time in seconds to sleep between fetching the blob list'
+    config_param :fetch_interval, :integer, default: 30
+    desc 'The the lease duration on the blob in seconds'
+    config_param :lease_duration, :integer, default: 60
+
+    def configure(conf)
+      super
+    end
+
+    def start
+      super
+      if @lease_duration > 60 || @lease_duration < 15
+        raise Fluent::ConfigError, "fluent-plugin-azure-queue: 'lease_duration' parameter must be between 15 and 60: #{@lease_duration}"
+      end
+      @blob_client = Azure::Storage::Client.create(
+        :storage_account_name => @storage_account_name,
+        :storage_access_key => @storage_access_key).blob_client
+      @running = true
+
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      log.debug("Begin in azure blob shutdown")
+      @running = false
+      @thread.join
+      log.debug("Finish in azure blob shutdown")
+      super
+    end
+
+    private
+
+    def run
+      log.debug("Begin running in azure blob")
+      @next_fetch_time = Time.now
+      while @running
+        if Time.now > @next_fetch_time
+          @next_fetch_time = Time.now + @fetch_interval
+          begin
+            blobs = @blob_client.list_blobs(@container_name)
+            blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
+            log.trace("Found #{blobs.count} unlocked blobs", container_name: @container_name)
+            # Blobs come back with oldest first
+            blobs.each do |blob|
+              ingest_blob(blob)
+            end
+          rescue => e
+            log.warn(error: e)
+            log.warn_backtrace(e.backtrace)
+          end
+        else
+          sleep(@next_fetch_time - Time.now)
+        end
+      end
+    end
+
+    def ingest_blob(blob)
+      begin
+        lease_id = @blob_client.acquire_blob_lease(@container_name, blob.name, duration: @lease_duration)
+        log.trace("Blob Leased", blob_name: blob.name)
+        blob, blob_contents = @blob_client.get_blob(@container_name, blob.name)
+        emit_blob_messages(blob_contents)
+        log.trace("Done Ingest blob", blob_name: blob.name)
+        begin
+          delete_blob(blob, lease_id)
+          log.debug("Blob deleted", blob_name: blob.name)
+        rescue Exception => e
+          log.warn("Records emmitted but blob not deleted", container_name: @container_name, blob_name: blob.name, error: e)
+          log.warn_backtrace(e.backtrace)
+        end
+      rescue Azure::Core::Http::HTTPError => e
+        if e.status_code == 409
+          log.info("Blob already leased", blob_name: blob.name)
+        else
+          log.warn("Error occurred while ingesting blob", error: e)
+          log.warn_backtrace(e.backtrace)
+        end
+      rescue Exception => e
+        log.warn("Error occurred while ingesting blob", error: e)
+        log.warn_backtrace(e.backtrace)
+      end
+    end
+
+    def emit_blob_messages(blob_contents)
+      buffer = StringIO.new(blob_contents)
+      reader = Avro::DataFile::Reader.new(buffer, Avro::IO::DatumReader.new)
+      event_stream = MultiEventStream.new
+      reader.each do |record|
+        time = Time.strptime(record["EnqueuedTimeUtc"], "%m/%d/%Y %r").to_i
+        value = { @message_key => record["Body"] }
+        event_stream.add(time, value)
+      end
+      router.emit_stream(@tag, event_stream)
+    end
+
+    def delete_blob(blob, lease_id)
+      # Hack because 'delete_blob' doesn't support lease_id yet
+      Azure::Storage::Service::StorageService.register_request_callback { |headers| headers["x-ms-lease-id"] = lease_id }
+      @blob_client.delete_blob(@container_name, blob.name)
+      Azure::Storage::Service::StorageService.register_request_callback { |headers| headers }
+    end
+  end
+end

data/lib/fluent/plugin/in_azure_queue.rb

@@ -1,6 +1,6 @@
 require 'fluent/input'
 require 'azure/storage'
-require 'base64'
+require 'concurrent'
 
 module Fluent
   class AzureQueueInput < Input
@@ -16,12 +16,10 @@ module Fluent
     config_param :queue_name, :string
     desc 'The the record key to put the message data into'
     config_param :message_key, :string, default: 'message'
-    desc 'The time in seconds to sleep between fetching messages'
-    config_param :fetch_interval, :integer, default: 5
-    desc 'The number of messages to fetch during each request'
-    config_param :batch_size, :integer, default: 10
     desc 'The the lease time on the messages in seconds'
-    config_param :lease_time, :integer, default: 30
+    config_param :lease_time, :integer, default: 300
+    desc 'The maximum number of threads to fetch messages'
+    config_param :max_fetch_threads, :integer, default: 30
 
     def configure(conf)
       super
@@ -29,15 +27,17 @@ module Fluent
 
     def start
       super
-      if @batch_size > 32 || @batch_size < 1
-        raise Fluent::ConfigError, "fluent-plugin-azure-queue: 'batch_size' parameter must be between 1 and 32: #{@batch_size}"
-      end
       @queue_client = Azure::Storage::Client.create(
         :storage_account_name => @storage_account_name,
         :storage_access_key => @storage_access_key).queue_client
       log.debug("Succeeded to creating azure queue client")
       @running = true
 
+      @delete_pool = Concurrent::ThreadPoolExecutor.new(
+        min_threads: 1,
+        max_threads: @max_fetch_threads,
+        max_queue: @max_fetch_threads,
+      )
       @thread = Thread.new(&method(:run))
     end
 
@@ -55,36 +55,27 @@ module Fluent
       log.debug("Begin running azure queue")
       @next_fetch_time = Time.now
       while @running
-        deleted = false
+        delete_futures = []
         begin
-          if Time.now > @next_fetch_time
-            @next_fetch_time += @fetch_interval
-            log.trace("Fetching #{@batch_size} messages from azure queue", queue_name: @queue_name)
-            messages = @queue_client.list_messages(@queue_name, @lease_time, { number_of_messages: @batch_size })
-            log.trace("Recieved #{messages.count} messages from azure queue", queue_name: @queue_name)
-            messages.each do |message|
-              deleted = false
-              message_content = Base64.decode64(message.message_text)
-              router.emit(@tag, Fluent::Engine.now, { @message_key => message_content })
-              log.trace("Emitted azure queue message", queue_name: @queue_name, id: message.id, content: message_content)
-              @queue_client.delete_message(@queue_name, message.id, message.pop_receipt)
-              log.trace("Deleted azure queue message", queue_name: @queue_name, id: message.id)
-              deleted = true
-            end
+          if ((@delete_pool.queue_length) < @max_fetch_threads)
+              Concurrent::Future.execute(executor: @delete_pool) do
+                start = Time.now
+                message = @queue_client.list_messages(@queue_name, @lease_time, { number_of_messages: 1 })[0]
+                log.trace("Recieved 1 messages from azure queue", queue_name: @queue_name, id: message.id, time: Time.now - start)
+                router.emit(@tag, Fluent::Engine.now, { @message_key => Base64.decode64(message.message_text)})
+                start = Time.now
+                @queue_client.delete_message(@queue_name, message.id, message.pop_receipt)
+                log.trace("Deleted azure queue message", queue_name: @queue_name, id: message.id, time: Time.now - start)
+              end
           else
-            sleep 0.05
+            log.trace("Not fetching more messages, already have #{@delete_pool.queue_length} messages to be deleted")
+            sleep 0.5
           end
         rescue => e
-          if deleted
-            log.warn(error: e)
-            log.warn_backtrace(e.backtrace)
-          else
-            log.warn("Message emmitted but not deleted from azure queue", queue_name: @queue_name, error: e)
-            log.warn_backtrace(e.backtrace)
-          end
+          log.warn(error: e)
+          log.warn_backtrace(e.backtrace)
         end
       end
-
     end
   end
 end

data/test/test_in_azure_event_hub_capture.rb

@@ -0,0 +1,103 @@
+require 'fluent/test'
+require 'fluent/plugin/in_azure_event_hub_capture'
+require 'flexmock/test_unit'
+require 'fluent/input'
+
+class AzureEventHubCaptureInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    if Fluent.const_defined?(:EventTime)
+      stub(Fluent::EventTime).now { @time }
+    end
+  end
+
+  CONFIG = %[
+    tag test_tag
+    storage_account_name test_storage_account_name
+    storage_access_key test_storage_access_key
+    container_name test_container_name
+    fetch_interval 1
+  ]
+
+  def create_driver(conf = CONFIG)
+    d = Fluent::Test::InputTestDriver.new(Fluent::AzureEventHubCaptureInput)
+    d.configure(conf)
+    d
+  end
+
+  Struct.new("Blob", :name, :properties)
+  
+  def test_configure
+    d = create_driver
+    assert_equal 'test_tag', d.instance.tag
+    assert_equal 'test_storage_account_name', d.instance.storage_account_name
+    assert_equal 'test_storage_access_key', d.instance.storage_access_key
+    assert_equal 'test_container_name', d.instance.container_name
+    assert_equal 1, d.instance.fetch_interval
+  end
+
+  def setup_mocks(driver)
+    blob_client = flexmock("blob_client")
+    client = flexmock("client", :blob_client => blob_client)
+    flexmock(Azure::Storage::Client, :create => client)
+    blob_client
+  end
+  
+  def test_no_blobs
+    d = create_driver
+    blob_client = setup_mocks(d)
+    blob_client.should_receive(:list_blobs).with(d.instance.container_name).and_return([]).once
+    flexmock(d.instance).should_receive(:ingest_blob).never()
+    d.run do
+      sleep 1
+    end
+  end
+
+  def test_two_blobs
+    d = create_driver
+    blobs = [Struct::Blob.new("test1", lease_status: "unlocked"), Struct::Blob.new("test2", lease_status: "unlocked")]
+    blob_client = setup_mocks(d)
+    blob_client.should_receive(:list_blobs).with(d.instance.container_name).and_return(blobs).once
+    plugin = flexmock(d.instance)
+    plugin.should_receive(:ingest_blob).with(blobs[0]).once()
+    plugin.should_receive(:ingest_blob).with(blobs[1]).once()
+    d.run do
+      sleep 1
+    end
+  end
+
+  def test_ingest_blob
+    d = create_driver
+    blob = Struct::Blob.new("test1", lease_status: "unlocked")
+    blob_client = setup_mocks(d)
+    plugin = flexmock(d.instance)
+    lease_id = "123"
+    blob_client.should_receive(:acquire_blob_lease).with(d.instance.container_name, blob.name, duration: d.instance.lease_duration).and_return(lease_id).once
+    updated_blob = Struct::Blob.new("test1", lease_status: "locked")
+    blob_contents = flexmock("blob_contents")
+    blob_client.should_receive(:get_blob).with(d.instance.container_name,  blob.name).and_return([updated_blob, blob_contents]).once
+    plugin.should_receive(:emit_blob_messages).with(blob_contents).once
+    plugin.should_receive(:delete_blob).with(updated_blob, lease_id).once
+    d.run do
+      plugin.send(:ingest_blob, blob)
+    end
+  end
+
+  def test_emit_blob_messages
+    d = create_driver
+    setup_mocks(d)
+    test_payload = flexmock("test_payload")
+    buffer = flexmock("buffer")
+    flexmock(StringIO).should_receive(:new).and_return(buffer)
+    time = 1504030204
+    time_string = Time.at(time).strftime("%m/%d/%Y %r")
+    original_payload = {"key" => "value"}.to_json
+    records = [ {"EnqueuedTimeUtc" => time_string, "Body" => original_payload } ] 
+    flexmock(Avro::DataFile::Reader).should_receive(:new).with(buffer, Avro::IO::DatumReader).and_return(records)
+    d.run do
+      d.instance.send(:emit_blob_messages, test_payload)
+    end
+    assert_equal(1, d.emits.size)
+    d.expect_emit(d.instance.tag, time, { "message" => original_payload })
+  end
+end

data/test/test_in_azure_queue.rb

@@ -40,25 +40,22 @@ class AzureQueueInputTest < Test::Unit::TestCase
     queue_client.should_receive(:list_messages).with(
       driver.instance.queue_name,
       driver.instance.lease_time,
-      { number_of_messages: driver.instance.batch_size}).and_return(messages).once
+      { number_of_messages: 1}).and_return(messages).once
     client = flexmock("client", :queue_client => queue_client)
     flexmock(Azure::Storage::Client, :create => client)
     queue_client
   end
 
-  def test_two_messages
+  def test_one_message
     d = create_driver
-    messages = [ Struct::QueueMessage.new(1, 99, Base64.encode64("test line")),
-                 Struct::QueueMessage.new(2, 100, Base64.encode64("test line2"))]
+    messages = [ Struct::QueueMessage.new(1, 99, Base64.encode64("test line"))]
     queue_client = setup_mocks(d, messages)
     queue_client.should_receive(:delete_message).with(d.instance.queue_name, messages[0].id,messages[0].pop_receipt).once
-    queue_client.should_receive(:delete_message).with(d.instance.queue_name, messages[1].id,messages[1].pop_receipt).once
     d.run do
       sleep 1
     end
-    assert_equal(2, d.emits.size)
+    assert_equal(1, d.emits.size)
     d.expect_emit(d.instance.tag, @time, { "message" => "test line" })
-    d.expect_emit(d.instance.tag, @time, { "message" => "test line2" })
   end
 
   def test_no_messages

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-queue
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre
+  version: 0.0.2.pre
 platform: ruby
 authors:
 - Scott Bonebrake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-22 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -64,6 +64,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: avro
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -121,7 +149,9 @@ files:
 - Rakefile
 - VERSION
 - fluent-plugin-azure-queue.gemspec
+- lib/fluent/plugin/in_azure_event_hub_capture.rb
 - lib/fluent/plugin/in_azure_queue.rb
+- test/test_in_azure_event_hub_capture.rb
 - test/test_in_azure_queue.rb
 homepage: https://github.com/sbonebrake/fluent-plugin-azure-queue
 licenses:
@@ -143,9 +173,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: Fluent input plugin for azure queue input
 test_files:
+- test/test_in_azure_event_hub_capture.rb
 - test/test_in_azure_queue.rb