fluent-plugin-azure-queue 0.0.8.pre → 0.0.11.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dec8e5dd6a0f5973f5741702060efb40cc04b22
-  data.tar.gz: 0b1fb55a244ccc5a8a1ad390d09f1950eaf3e22c
+  metadata.gz: 37078952b1ed7c05bf93e680880cefb7972f7bcd
+  data.tar.gz: 92a27fe6426bdcb45af5376ee63c071fc786021c
 SHA512:
-  metadata.gz: d4cf9f3f62e6213c4aeb421741bcde86fc4ecffef9a0b0b3efa0c0002390cb58e398c8187d698bab9b9f0babd01c531ea5a0273877cfbad7bbdb74cf505e4555
-  data.tar.gz: 6092c42d2fba01796c75085c1692e4f94696f37ba7314fdf38273d3cc309d6187fcf4562b44a46df7203f7b9f3dc865e0bb304248f0fa39d3bc9c568eb378e98
+  metadata.gz: 8b35295865bc0d3703a843cfb49d0c751ad2186dda249861b108c1af36d65381d8d97e671923ea9bdbe47b215a0886411f8d71ea484b898f661029623bc5d63b
+  data.tar.gz: 08cc210e8ae481c643b9614ffa71087a8f0867b2d00b40d40d877b1d8a8f727cfb058c2385badb75734ec25e93d0dc6d2f14b9f0779e9af76474f68e80c128d6

data/VERSION

@@ -1 +1 @@
-0.0.8.pre
+0.0.11.pre

data/fluent-plugin-azure-queue.gemspec

@@ -16,7 +16,8 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", [">= 0.12.2", "< 0.14"]
-  gem.add_dependency "azure-storage", [">= 0.12.3.preview", "< 0.13"]
+  gem.add_dependency "azure-storage-blob", [">= 1.0.1", "< 1.1.0"]
+  gem.add_dependency "azure-storage-queue", [">= 1.0.1", "< 1.1.0"]
   gem.add_dependency "nokogiri"
   gem.add_dependency "concurrent-ruby"
   gem.add_dependency "avro", ">= 1.8"

data/lib/fluent/plugin/in_azure_event_hub_capture.rb

@@ -1,5 +1,7 @@
 require 'fluent/input'
-require 'azure/storage'
+require 'azure/storage/common'
+require "azure/storage/blob"
+require "azure/storage/queue"
 require "avro"
 
 module Fluent
@@ -34,9 +36,11 @@ module Fluent
       if @lease_duration > 60 || @lease_duration < 15
         raise Fluent::ConfigError, "fluent-plugin-azure-queue: 'lease_duration' parameter must be between 15 and 60: #{@lease_duration}"
       end
-      @azure_client = Azure::Storage::Client.create(
+      azure_client = Azure::Storage::Common::Client.create(
         :storage_account_name => @storage_account_name,
         :storage_access_key => @storage_access_key)
+      @blob_client = Azure::Storage::Blob::BlobService.new(client: azure_client)
+      @queue_client = Azure::Storage::Queue::QueueService.new(client: azure_client)
       @running = true
       @containers = container_names.split(',').map { |c| c.strip }
 
@@ -73,7 +77,7 @@ module Fluent
     def ingest_from_blob_list
       @containers.each do |container_name|
         begin
-          blobs = @azure_client.blob_client.list_blobs(container_name)
+          blobs = @blob_client.list_blobs(container_name)
           blobs = blobs.select { |b| b.properties[:lease_status] == "unlocked" }
           log.info("Found #{blobs.count} unlocked blobs", container_name: container_name)
           # Blobs come back with oldest first
@@ -89,11 +93,11 @@ module Fluent
 
     def ingest_from_queue
       begin
-        blob_id_messages = @azure_client.queue_client.list_messages(@queue_name, @queue_lease_time, { number_of_messages: 32 })
+        blob_id_messages = @queue_client.list_messages(@queue_name, @queue_lease_time, { number_of_messages: 32 })
         blob_id_messages.each do |blob_id_message|
           blob_id = JSON.parse(Base64.decode64(blob_id_message.message_text))
           ingest_blob(blob_id["Container"], blob_id["Name"])
-          @azure_client.queue_client.delete_message(@queue_name, blob_id_message.id, blob_id_message.pop_receipt)
+          @queue_client.delete_message(@queue_name, blob_id_message.id, blob_id_message.pop_receipt)
         end
       rescue => e
         log.warn(error: e)
@@ -103,13 +107,13 @@ module Fluent
 
     def ingest_blob(container_name, blob_name)
       begin
-        lease_id = @azure_client.blob_client.acquire_blob_lease(container_name, blob_name, duration: @lease_duration)
+        lease_id = @blob_client.acquire_blob_lease(container_name, blob_name, duration: @lease_duration)
         log.info("Blob Leased", blob_name: blob_name)
-        blob, blob_contents = @azure_client.blob_client.get_blob(container_name, blob_name)
+        blob, blob_contents = @blob_client.get_blob(container_name, blob_name)
         emit_blob_messages(blob_contents)
         log.trace("Done Ingest blob", blob_name: blob_name)
         begin
-          delete_blob(container_name, blob, lease_id)
+          @blob_client.delete_blob(container_name, blob_name, lease_id: lease_id)
           log.debug("Blob deleted", blob_name: blob_name)
         rescue Exception => e
           log.warn("Records emmitted but blob not deleted", container_name: container_name, blob_name: blob_name, error: e)
@@ -149,12 +153,5 @@ module Fluent
       end
       router.emit_stream(@tag, event_stream)
     end
-
-    def delete_blob(container_name, blob, lease_id)
-      # Hack because 'delete_blob' doesn't support lease_id yet
-      Azure::Storage::Service::StorageService.register_request_callback { |headers| headers["x-ms-lease-id"] = lease_id }
-      @azure_client.blob_client.delete_blob(container_name, blob.name)
-      Azure::Storage::Service::StorageService.register_request_callback { |headers| headers }
-    end
   end
 end

data/lib/fluent/plugin/in_azure_queue.rb

@@ -1,5 +1,5 @@
 require 'fluent/input'
-require 'azure/storage'
+require 'azure/storage/queue'
 require 'concurrent'
 
 module Fluent
@@ -27,9 +27,9 @@ module Fluent
 
     def start
       super
-      @queue_client = Azure::Storage::Client.create(
+      @queue_client = Azure::Storage::Queue::QueueService.create(
         :storage_account_name => @storage_account_name,
-        :storage_access_key => @storage_access_key).queue_client
+        :storage_access_key => @storage_access_key)
       log.debug("Succeeded to creating azure queue client")
       @running = true
 

data/test/test_in_azure_event_hub_capture.rb

@@ -50,8 +50,9 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
   def setup_mocks(driver)
     blob_client = flexmock("blob_client")
     queue_client = flexmock("queue_client")
-    client = flexmock("client", :blob_client => blob_client, :queue_client => queue_client)
-    flexmock(Azure::Storage::Client, :create => client)
+    flexmock(Azure::Storage::Common::Client, :create => nil)
+    flexmock(Azure::Storage::Queue::QueueService, :new => queue_client)
+    flexmock(Azure::Storage::Blob::BlobService, :new => blob_client)
     [blob_client, queue_client]
   end
 
@@ -71,8 +72,8 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     blob_client, queue_client = setup_mocks(d)
     blob_client.should_receive(:list_blobs).with(d.instance.container_names).and_return(blobs).once
     plugin = flexmock(d.instance)
-    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[0]).once()
-    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[1]).once()
+    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[0].name).once()
+    plugin.should_receive(:ingest_blob).with(d.instance.container_names, blobs[1].name).once()
     d.run do
       sleep 1
     end
@@ -121,11 +122,11 @@ class AzureEventHubCaptureInputTest < Test::Unit::TestCase
     blob_client.should_receive(:acquire_blob_lease).with(d.instance.container_names, blob.name, duration: d.instance.lease_duration).and_return(lease_id).once
     updated_blob = Struct::Blob.new("test1", lease_status: "locked")
     blob_contents = flexmock("blob_contents")
-    blob_client.should_receive(:get_blob).with(d.instance.container_names,  blob.name).and_return([updated_blob, blob_contents]).once
+    blob_client.should_receive(:get_blob).with(d.instance.container_names, blob.name).and_return([updated_blob, blob_contents]).once
     plugin.should_receive(:emit_blob_messages).with(blob_contents).once
-    plugin.should_receive(:delete_blob).with(d.instance.container_names, updated_blob, lease_id).once
+    blob_client.should_receive(:delete_blob).with(d.instance.container_names, updated_blob.name, lease_id: lease_id).once
     d.run do
-      plugin.send(:ingest_blob, d.instance.container_names, blob)
+      plugin.send(:ingest_blob, d.instance.container_names, blob.name)
     end
   end
 

data/test/test_in_azure_queue.rb

@@ -41,8 +41,7 @@ class AzureQueueInputTest < Test::Unit::TestCase
       driver.instance.queue_name,
       driver.instance.lease_time,
       { number_of_messages: 1}).and_return(messages).once
-    client = flexmock("client", :queue_client => queue_client)
-    flexmock(Azure::Storage::Client, :create => client)
+    flexmock(Azure::Storage::Queue::QueueService, :create => queue_client)
     queue_client
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-queue
 version: !ruby/object:Gem::Version
-  version: 0.0.8.pre
+  version: 0.0.11.pre
 platform: ruby
 authors:
 - Scott Bonebrake
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-19 00:00:00.000000000 Z
+date: 2018-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -31,25 +31,45 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.14'
 - !ruby/object:Gem::Dependency
-  name: azure-storage
+  name: azure-storage-blob
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.3.preview
+        version: 1.0.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.3.preview
+        version: 1.0.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.13'
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: azure-storage-queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -172,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Fluent input plugin for azure queue input