fluent-plugin-azure-queue 0.0.5.pre → 0.0.6.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ce24e4937db26a3918accbc66d8b39692a1259c
-  data.tar.gz: b9a068ff1a161c12e1f80cd8a6cb8ac716936dae
+  metadata.gz: b9216faa1348ea85dd618fcc498b31efea22eac1
+  data.tar.gz: 18989b0f648643a7f29d06f51c4b2d49b3850132
 SHA512:
-  metadata.gz: 990edfb1e19407bf039dd05de4d388c4f54d65dd911712ccbac0a884bbe9a66657757448cf139b72f0357373f8c96e9a3882ac7a96a1a634099a1783b28f26de
-  data.tar.gz: ecedf28bd3eced34c4405805c9f1845393b43ce7d15ee96a750231e3f6c00442f2b955ac0e0ecde51e17ffed7e38234415478cd22e217850c0620c4c20f126d8
+  metadata.gz: 9e721814bfb21da863ae2ec607743967a398c79719ba03330b7842f65cd8b27eb7e7968e685dabcd710528f5df3bcad3a2ef508507f481002aa51ebe860ed0a2
+  data.tar.gz: 3d2ce94dcd4f395a27e76c2bc7fa1605978914def55ee44d18b113cdbd204e31e70ad17396b38742e7ee65b2f1565a236ec7739e61abee605113ca108271702f

data/README.md

@@ -20,7 +20,7 @@ fluentd v.12
       storage_account_name my_storage_account
       storage_access_key my_storage_access_key
       queue_name my_storage_queue
-      fetch_interval 5     
+      fetch_interval 5
       lease_duration 30
     </source>
 
@@ -69,7 +69,7 @@ public static void Run(string[] hubMessages, ICollector<string> outputQueue, Tra
         {
             outputQueue.Add(message);
         }
-        else 
+        else
         {
             log.Warning($"Message is larger than 64k with {bytes} bytes. Dropping message");
         }
@@ -77,7 +77,7 @@ public static void Run(string[] hubMessages, ICollector<string> outputQueue, Tra
 }
 ```
 ## azure_event_hub_capture Input Plugin
-This plugin is designed to work with blobs stored to a container via [Azure Event Hubs Capture](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview)  
+This plugin is designed to work with blobs stored to a container via [Azure Event Hubs Capture](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview)
 
 **Warning:** This plugin will delete the blobs after emitting the contents in fluentd.
 
@@ -89,8 +89,8 @@ This plugin is designed to work with blobs stored to a container via [Azure Even
       tag event_hub_input
       storage_account_name my_storage_account
       storage_access_key my_storage_access_key
-      container_name my_capture_container
-      fetch_interval 30     
+      container_names my_capture_container
+      fetch_interval 30
       lease_duration 30
     </source>
 
@@ -106,9 +106,9 @@ The storage account name
 
 The storage account access key
 
-**container_name (required)**
+**container_names (required)**
 
-The capture container name
+The capture container names, comma separated.
 
 **message_key**
 

data/VERSION

@@ -1 +1 @@
-0.0.5.pre
+0.0.6.pre

data/lib/fluent/plugin/in_azure_event_hub_capture.rb

@@ -12,8 +12,8 @@ module Fluent
     config_param :storage_account_name, :string
     desc 'The azure storage account access key'
     config_param :storage_access_key, :string
-    desc 'The container name'
-    config_param :container_name, :string
+    desc 'The container name(s). Use commas to separate'
+    config_param :container_names, :string
     desc 'The the record key to put the message data into'
     config_param :message_key, :string, default: 'message'
     desc 'The time in seconds to sleep between fetching the blob list'
@@ -34,6 +34,7 @@ module Fluent
         :storage_account_name => @storage_account_name,
         :storage_access_key => @storage_access_key).blob_client
       @running = true
+      @containers = container_names.split(',').map { |c| c.strip }
 
       @thread = Thread.new(&method(:run))
     end
@@ -54,17 +55,19 @@ module Fluent
       while @running
         if Time.now > @next_fetch_time
           @next_fetch_time = Time.now + @fetch_interval
-          begin
-            blobs = @blob_client.list_blobs(@container_name)
-            blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
-            log.info("Found #{blobs.count} unlocked blobs", container_name: @container_name)
-            # Blobs come back with oldest first
-            blobs.each do |blob|
-              ingest_blob(blob)
+          @containers.each do |container_name|
+            begin
+              blobs = @blob_client.list_blobs(container_name)
+              blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
+              log.info("Found #{blobs.count} unlocked blobs", container_name: container_name)
+              # Blobs come back with oldest first
+              blobs.each do |blob|
+                ingest_blob(container_name, blob)
+              end
+            rescue => e
+              log.warn(error: e)
+              log.warn_backtrace(e.backtrace)
             end
-          rescue => e
-            log.warn(error: e)
-            log.warn_backtrace(e.backtrace)
           end
         else
           sleep(@next_fetch_time - Time.now)
@@ -72,18 +75,18 @@ module Fluent
       end
     end
 
-    def ingest_blob(blob)
+    def ingest_blob(container_name, blob)
       begin
-        lease_id = @blob_client.acquire_blob_lease(@container_name, blob.name, duration: @lease_duration)
+        lease_id = @blob_client.acquire_blob_lease(container_name, blob.name, duration: @lease_duration)
         log.info("Blob Leased", blob_name: blob.name)
-        blob, blob_contents = @blob_client.get_blob(@container_name, blob.name)
+        blob, blob_contents = @blob_client.get_blob(container_name, blob.name)
         emit_blob_messages(blob_contents)
         log.trace("Done Ingest blob", blob_name: blob.name)
         begin
-          delete_blob(blob, lease_id)
+          delete_blob(container_name, blob, lease_id)
           log.debug("Blob deleted", blob_name: blob.name)
         rescue Exception => e
-          log.warn("Records emmitted but blob not deleted", container_name: @container_name, blob_name: blob.name, error: e)
+          log.warn("Records emmitted but blob not deleted", container_name: container_name, blob_name: blob.name, error: e)
           log.warn_backtrace(e.backtrace)
         end
       rescue Azure::Core::Http::HTTPError => e
@@ -121,10 +124,10 @@ module Fluent
       router.emit_stream(@tag, event_stream)
     end
 
-    def delete_blob(blob, lease_id)
+    def delete_blob(container_name, blob, lease_id)
       # Hack because 'delete_blob' doesn't support lease_id yet
       Azure::Storage::Service::StorageService.register_request_callback { |headers| headers["x-ms-lease-id"] = lease_id }
-      @blob_client.delete_blob(@container_name, blob.name)
+      @blob_client.delete_blob(container_name, blob.name)
       Azure::Storage::Service::StorageService.register_request_callback { |headers| headers }
     end
   end

data/test/test_in_azure_event_hub_capture.rb

@@ -15,7 +15,7 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     tag test_tag
     storage_account_name test_storage_account_name
     storage_access_key test_storage_access_key
-    container_name test_container_name
+    container_names test_container_name
     fetch_interval 1
   ]
 
@@ -26,13 +26,13 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
   end
 
   Struct.new("Blob", :name, :properties)
-  
+
   def test_configure
     d = create_driver
     assert_equal 'test_tag', d.instance.tag
     assert_equal 'test_storage_account_name', d.instance.storage_account_name
     assert_equal 'test_storage_access_key', d.instance.storage_access_key
-    assert_equal 'test_container_name', d.instance.container_name
+    assert_equal 'test_container_name', d.instance.container_names
     assert_equal 1, d.instance.fetch_interval
   end
 
@@ -42,11 +42,11 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     flexmock(Azure::Storage::Client, :create => client)
     blob_client
   end
-  
+
   def test_no_blobs
     d = create_driver
     blob_client = setup_mocks(d)
-    blob_client.should_receive(:list_blobs).with(d.instance.container_name).and_return([]).once
+    blob_client.should_receive(:list_blobs).with(d.instance.container_names).and_return([]).once
     flexmock(d.instance).should_receive(:ingest_blob).never()
     d.run do
       sleep 1
@@ -57,10 +57,10 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     d = create_driver
     blobs = [Struct::Blob.new("test1", lease_status: "unlocked"), Struct::Blob.new("test2", lease_status: "unlocked")]
     blob_client = setup_mocks(d)
-    blob_client.should_receive(:list_blobs).with(d.instance.container_name).and_return(blobs).once
+    blob_client.should_receive(:list_blobs).with(d.instance.container_names).and_return(blobs).once
     plugin = flexmock(d.instance)
-    plugin.should_receive(:ingest_blob).with(blobs[0]).once()
-    plugin.should_receive(:ingest_blob).with(blobs[1]).once()
+    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[0]).once()
+    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[1]).once()
     d.run do
       sleep 1
     end
@@ -72,14 +72,14 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     blob_client = setup_mocks(d)
     plugin = flexmock(d.instance)
     lease_id = "123"
-    blob_client.should_receive(:acquire_blob_lease).with(d.instance.container_name, blob.name, duration: d.instance.lease_duration).and_return(lease_id).once
+    blob_client.should_receive(:acquire_blob_lease).with(d.instance.container_names, blob.name, duration: d.instance.lease_duration).and_return(lease_id).once
     updated_blob = Struct::Blob.new("test1", lease_status: "locked")
     blob_contents = flexmock("blob_contents")
-    blob_client.should_receive(:get_blob).with(d.instance.container_name,  blob.name).and_return([updated_blob, blob_contents]).once
+    blob_client.should_receive(:get_blob).with(d.instance.container_names,  blob.name).and_return([updated_blob, blob_contents]).once
     plugin.should_receive(:emit_blob_messages).with(blob_contents).once
-    plugin.should_receive(:delete_blob).with(updated_blob, lease_id).once
+    plugin.should_receive(:delete_blob).with(d.instance.container_names, updated_blob, lease_id).once
     d.run do
-      plugin.send(:ingest_blob, blob)
+      plugin.send(:ingest_blob, d.instance.container_names, blob)
     end
   end
 
@@ -92,7 +92,7 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     time = 1504030204
     time_string = Time.at(time).strftime("%m/%d/%Y %r")
     original_payload = {"key" => "value"}.to_json
-    records = [ {"EnqueuedTimeUtc" => time_string, "Body" => original_payload } ] 
+    records = [ {"EnqueuedTimeUtc" => time_string, "Body" => original_payload } ]
     flexmock(Avro::DataFile::Reader).should_receive(:new).with(buffer, Avro::IO::DatumReader).and_return(records)
     d.run do
       d.instance.send(:emit_blob_messages, test_payload)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-queue
 version: !ruby/object:Gem::Version
-  version: 0.0.5.pre
+  version: 0.0.6.pre
 platform: ruby
 authors:
 - Scott Bonebrake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-08 00:00:00.000000000 Z
+date: 2017-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd