fluent-plugin-azure-queue 0.0.6.pre → 0.0.8.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9216faa1348ea85dd618fcc498b31efea22eac1
-  data.tar.gz: 18989b0f648643a7f29d06f51c4b2d49b3850132
+  metadata.gz: 7dec8e5dd6a0f5973f5741702060efb40cc04b22
+  data.tar.gz: 0b1fb55a244ccc5a8a1ad390d09f1950eaf3e22c
 SHA512:
-  metadata.gz: 9e721814bfb21da863ae2ec607743967a398c79719ba03330b7842f65cd8b27eb7e7968e685dabcd710528f5df3bcad3a2ef508507f481002aa51ebe860ed0a2
-  data.tar.gz: 3d2ce94dcd4f395a27e76c2bc7fa1605978914def55ee44d18b113cdbd204e31e70ad17396b38742e7ee65b2f1565a236ec7739e61abee605113ca108271702f
+  metadata.gz: d4cf9f3f62e6213c4aeb421741bcde86fc4ecffef9a0b0b3efa0c0002390cb58e398c8187d698bab9b9f0babd01c531ea5a0273877cfbad7bbdb74cf505e4555
+  data.tar.gz: 6092c42d2fba01796c75085c1692e4f94696f37ba7314fdf38273d3cc309d6187fcf4562b44a46df7203f7b9f3dc865e0bb304248f0fa39d3bc9c568eb378e98

data/README.md

@@ -76,6 +76,7 @@ public static void Run(string[] hubMessages, ICollector<string> outputQueue, Tra
     }
 }
 ```
+
 ## azure_event_hub_capture Input Plugin
 This plugin is designed to work with blobs stored to a container via [Azure Event Hubs Capture](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview)
 
@@ -121,3 +122,37 @@ The time in seconds to sleep between fetching the blob list. Default 30
 **lease_duration**
 
 The time to lease the messages for. Default 60
+
+**queue_lease_time**
+
+The time to lease the queue message for. Default 60
+
+**queue_name**
+
+If set, get the blob names from a storage queue. Good if there are many blobs because listblobs is not reliable.
+
+## Integration with Azure Event Hub
+
+Use this azure function if you want to ingest event hub capture blob names from a storage queue.
+
+```c#
+// Blob trigger binding to a CloudBlockBlob
+#r "Microsoft.WindowsAzure.Storage"
+
+using Microsoft.WindowsAzure.Storage.Blob;
+
+public static void Run(CloudBlockBlob myBlob, out BlobIdentifier outputQueueItem, TraceWriter log)
+{
+    log.Info($"C# Blob trigger function Processed blob Name:{myBlob.Name} URI:{myBlob.StorageUri}");
+    BlobIdentifier blobId = new BlobIdentifier { Name = myBlob.Name, Container = myBlob.Container.Name };
+    //outputQueueItem.add(blobId);
+    outputQueueItem = blobId;
+}
+
+
+public class BlobIdentifier
+{
+    public string Container { get; set; }
+    public string Name { get; set; }
+}
+```

data/VERSION

@@ -1 +1 @@
-0.0.6.pre
+0.0.8.pre

data/lib/fluent/plugin/in_azure_event_hub_capture.rb

@@ -20,6 +20,10 @@ module Fluent
     config_param :fetch_interval, :integer, default: 30
     desc 'The the lease duration on the blob in seconds'
     config_param :lease_duration, :integer, default: 60
+    desc 'The the lease time on the messages in seconds'
+    config_param :queue_lease_time, :integer, default: 60
+    desc 'If set, get the blob names from a queue rather than the "list blobs" operation'
+    config_param :queue_name, :string, default: nil
 
     def configure(conf)
       super
@@ -30,9 +34,9 @@ module Fluent
       if @lease_duration > 60 || @lease_duration < 15
         raise Fluent::ConfigError, "fluent-plugin-azure-queue: 'lease_duration' parameter must be between 15 and 60: #{@lease_duration}"
       end
-      @blob_client = Azure::Storage::Client.create(
+      @azure_client = Azure::Storage::Client.create(
         :storage_account_name => @storage_account_name,
-        :storage_access_key => @storage_access_key).blob_client
+        :storage_access_key => @storage_access_key)
       @running = true
       @containers = container_names.split(',').map { |c| c.strip }
 
@@ -55,19 +59,10 @@ module Fluent
       while @running
         if Time.now > @next_fetch_time
           @next_fetch_time = Time.now + @fetch_interval
-          @containers.each do |container_name|
-            begin
-              blobs = @blob_client.list_blobs(container_name)
-              blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
-              log.info("Found #{blobs.count} unlocked blobs", container_name: container_name)
-              # Blobs come back with oldest first
-              blobs.each do |blob|
-                ingest_blob(container_name, blob)
-              end
-            rescue => e
-              log.warn(error: e)
-              log.warn_backtrace(e.backtrace)
-            end
+          if queue_name
+            ingest_from_queue
+          else
+            ingest_from_blob_list
           end
         else
           sleep(@next_fetch_time - Time.now)
@@ -75,25 +70,56 @@ module Fluent
       end
     end
 
-    def ingest_blob(container_name, blob)
+    def ingest_from_blob_list
+      @containers.each do |container_name|
+        begin
+          blobs = @azure_client.blob_client.list_blobs(container_name)
+          blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
+          log.info("Found #{blobs.count} unlocked blobs", container_name: container_name)
+          # Blobs come back with oldest first
+          blobs.each do |blob|
+            ingest_blob(container_name, blob.name)
+          end
+        rescue => e
+          log.warn(error: e)
+          log.warn_backtrace(e.backtrace)
+        end
+      end
+    end
+
+    def ingest_from_queue
+      begin
+        blob_id_messages = @azure_client.queue_client.list_messages(@queue_name, @queue_lease_time, { number_of_messages: 32 })
+        blob_id_messages.each do |blob_id_message|
+          blob_id = JSON.parse(Base64.decode64(blob_id_message.message_text))
+          ingest_blob(blob_id["Container"], blob_id["Name"])
+          @azure_client.queue_client.delete_message(@queue_name, blob_id_message.id, blob_id_message.pop_receipt)
+        end
+      rescue => e
+        log.warn(error: e)
+        log.warn_backtrace(e.backtrace)
+      end
+    end
+
+    def ingest_blob(container_name, blob_name)
       begin
-        lease_id = @blob_client.acquire_blob_lease(container_name, blob.name, duration: @lease_duration)
-        log.info("Blob Leased", blob_name: blob.name)
-        blob, blob_contents = @blob_client.get_blob(container_name, blob.name)
+        lease_id = @azure_client.blob_client.acquire_blob_lease(container_name, blob_name, duration: @lease_duration)
+        log.info("Blob Leased", blob_name: blob_name)
+        blob, blob_contents = @azure_client.blob_client.get_blob(container_name, blob_name)
         emit_blob_messages(blob_contents)
-        log.trace("Done Ingest blob", blob_name: blob.name)
+        log.trace("Done Ingest blob", blob_name: blob_name)
         begin
           delete_blob(container_name, blob, lease_id)
-          log.debug("Blob deleted", blob_name: blob.name)
+          log.debug("Blob deleted", blob_name: blob_name)
         rescue Exception => e
-          log.warn("Records emmitted but blob not deleted", container_name: container_name, blob_name: blob.name, error: e)
+          log.warn("Records emmitted but blob not deleted", container_name: container_name, blob_name: blob_name, error: e)
           log.warn_backtrace(e.backtrace)
         end
       rescue Azure::Core::Http::HTTPError => e
         if e.status_code == 409
-          log.trace("Blob already leased", blob_name: blob.name)
+          log.trace("Blob already leased", blob_name: blob_name)
         elsif e.status_code == 404
-          log.trace("Blob already deleted", blob_name: blob.name)
+          log.trace("Blob already deleted", blob_name: blob_name)
         else
           log.warn("Error occurred while ingesting blob", error: e)
           log.warn_backtrace(e.backtrace)
@@ -127,7 +153,7 @@ module Fluent
     def delete_blob(container_name, blob, lease_id)
       # Hack because 'delete_blob' doesn't support lease_id yet
       Azure::Storage::Service::StorageService.register_request_callback { |headers| headers["x-ms-lease-id"] = lease_id }
-      @blob_client.delete_blob(container_name, blob.name)
+      @azure_client.blob_client.delete_blob(container_name, blob.name)
       Azure::Storage::Service::StorageService.register_request_callback { |headers| headers }
     end
   end

data/test/test_in_azure_event_hub_capture.rb

@@ -11,7 +11,7 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     end
   end
 
-  CONFIG = %[
+  LIST_CONFIG = %[
     tag test_tag
     storage_account_name test_storage_account_name
     storage_access_key test_storage_access_key
@@ -19,13 +19,24 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     fetch_interval 1
   ]
 
-  def create_driver(conf = CONFIG)
+  QUEUE_CONFIG = %[
+    tag test_tag
+    storage_account_name test_storage_account_name
+    storage_access_key test_storage_access_key
+    container_names test_container_name
+    fetch_interval 1
+    queue_lease_time 30
+    queue_name test_queue_name
+  ]
+
+  def create_driver(conf = LIST_CONFIG)
     d = Fluent::Test::InputTestDriver.new(Fluent::AzureEventHubCaptureInput)
     d.configure(conf)
     d
   end
 
   Struct.new("Blob", :name, :properties)
+  Struct.new("QueueMessage", :id, :pop_receipt, :message_text)
 
   def test_configure
     d = create_driver
@@ -38,14 +49,15 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
 
   def setup_mocks(driver)
     blob_client = flexmock("blob_client")
-    client = flexmock("client", :blob_client => blob_client)
+    queue_client = flexmock("queue_client")
+    client = flexmock("client", :blob_client => blob_client, :queue_client => queue_client)
     flexmock(Azure::Storage::Client, :create => client)
-    blob_client
+    [blob_client, queue_client]
   end
 
-  def test_no_blobs
+  def test_list_no_blobs
     d = create_driver
-    blob_client = setup_mocks(d)
+    blob_client, queue_client = setup_mocks(d)
     blob_client.should_receive(:list_blobs).with(d.instance.container_names).and_return([]).once
     flexmock(d.instance).should_receive(:ingest_blob).never()
     d.run do
@@ -53,10 +65,10 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     end
   end
 
-  def test_two_blobs
+  def test_list_two_blobs
     d = create_driver
     blobs = [Struct::Blob.new("test1", lease_status: "unlocked"), Struct::Blob.new("test2", lease_status: "unlocked")]
-    blob_client = setup_mocks(d)
+    blob_client, queue_client = setup_mocks(d)
     blob_client.should_receive(:list_blobs).with(d.instance.container_names).and_return(blobs).once
     plugin = flexmock(d.instance)
     plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[0]).once()
@@ -66,10 +78,44 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_queue_no_blobs
+    d = create_driver(QUEUE_CONFIG)
+    blob_client, queue_client = setup_mocks(d)
+    queue_client.should_receive(:list_messages).with(
+      d.instance.queue_name,
+      d.instance.queue_lease_time,
+      { number_of_messages: 32}).and_return([]).once
+    flexmock(d.instance).should_receive(:ingest_blob).never()
+    d.run do
+      sleep 1
+    end
+  end
+
+  def test_queue_two_blobs
+    d = create_driver(QUEUE_CONFIG)
+    blob_id1 = { "Name" => "test1", "Container" => "container1" }
+    blob_id2 = { "Name" => "test2", "Container" => "container2" }
+    blobs_id_messages = [ Struct::QueueMessage.new(1, 99, Base64.encode64(blob_id1.to_json)),
+                          Struct::QueueMessage.new(2, 299, Base64.encode64(blob_id2.to_json))]
+    blob_client, queue_client = setup_mocks(d)
+    queue_client.should_receive(:list_messages).with(
+      d.instance.queue_name,
+      d.instance.queue_lease_time,
+      { number_of_messages: 32}).and_return(blobs_id_messages).once
+    plugin = flexmock(d.instance)
+    plugin.should_receive(:ingest_blob).with(blob_id1["Container"], blob_id1["Name"]).once()
+    plugin.should_receive(:ingest_blob).with(blob_id2["Container"], blob_id2["Name"]).once()
+    queue_client.should_receive(:delete_message).with(d.instance.queue_name, blobs_id_messages[0].id, blobs_id_messages[0].pop_receipt).once
+    queue_client.should_receive(:delete_message).with(d.instance.queue_name, blobs_id_messages[1].id, blobs_id_messages[1].pop_receipt).once
+    d.run do
+      sleep 1
+    end
+  end
+
   def test_ingest_blob
     d = create_driver
     blob = Struct::Blob.new("test1", lease_status: "unlocked")
-    blob_client = setup_mocks(d)
+    blob_client, queue_client = setup_mocks(d)
     plugin = flexmock(d.instance)
     lease_id = "123"
     blob_client.should_receive(:acquire_blob_lease).with(d.instance.container_names, blob.name, duration: d.instance.lease_duration).and_return(lease_id).once
@@ -85,7 +131,7 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
 
   def test_emit_blob_messages
     d = create_driver
-    setup_mocks(d)
+    blob_client, queue_client = setup_mocks(d)
     test_payload = flexmock("test_payload")
     buffer = flexmock("buffer")
     flexmock(StringIO).should_receive(:new).and_return(buffer)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-queue
 version: !ruby/object:Gem::Version
-  version: 0.0.6.pre
+  version: 0.0.8.pre
 platform: ruby
 authors:
 - Scott Bonebrake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-20 00:00:00.000000000 Z
+date: 2017-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd