fluent-plugin-azurestorage 0.0.8 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 776476b29eb416de1bf483aaf7f2a7a864ed10fe
-  data.tar.gz: 95cdf6fa62d1f4e772179694a1a3f7a611ee3149
+  metadata.gz: 19fc61140e33ed5db2e6f41736d8a41515fd2eab
+  data.tar.gz: c83005cb236f4b19a8c547945930e4783bd02366
 SHA512:
-  metadata.gz: cec4a7f65d722e23bb49e46211c482b10f7c3e9f15e0ed286b960599aac7034a5d110db122ab072ace1ea056a820b68d7e2f55a5b9033f6252a6ba71710718cc
-  data.tar.gz: b21756a7d13066e1155853ee3ed2bd9262ef24304483f0306c509f34538ae4ee9029efd0a1d161432f65520785c30e7033f35aeee9e10a68acaca710bc0cd6ad
+  metadata.gz: ca632bb038f91cf996ff63bb171bdda0e67baee3c6911513ef4f884862cb2fd0ea17161273518134eb952eb406b2561ae1b07968466e1cb190b112c7a7bbd65a
+  data.tar.gz: 12c019aa3ef994cd7089f5fe419e5ffd301226429054a85f18b94e3abf24b157c2427854755f772fa27fd726d70632fdbf21178c61fce7950253fb20bd36309f

data/.gitignore

@@ -9,3 +9,4 @@
 Gemfile.lock
 vendor
 .ruby-version
+*.komodoproject

data/README.md

@@ -16,6 +16,48 @@ $ gem install fluent-plugin-azurestorage
 
 ## Configuration
 
+### v0.14 style
+
+```
+<match pattern>
+  type azurestorage
+
+  azure_storage_account    <your azure storage account>
+  azure_storage_access_key <your azure storage access key>
+  azure_container          <your azure storage container>
+  azure_storage_type       blob
+  store_as                 gzip
+  auto_create_container    true
+  path                     logs/
+  azure_object_key_format  %{path}%{time_slice}_%{index}.%{file_extension}
+  time_slice_format        %Y%m%d-%H
+  # if you want to use ${tag} or %Y/%m/%d/ like syntax in path / s3_object_key_format,
+  # need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/azurestorage
+    timekey 3600 # 1 hour partition
+    timekey_wait 10m
+    timekey_use_utc true # use utc
+  </buffer>
+</match>
+```
+
+For `<buffer>`, you can use any record field in `path` / `azure_object_key_format`.
+
+```
+path logs/${tag}/${foo}
+<buffer tag,foo>
+  # parameters...
+</buffer>
+```
+
+See official article for more detail: Buffer section configurations
+
+Note that this configuration doesn't work with fluentd v0.12.
+
+### v0.12 style
+
 ```
 <match pattern>
   type azurestorage

data/VERSION

@@ -1 +1 @@
-0.0.8
+0.1.0

data/fluent-plugin-azurestorage.gemspec

@@ -17,9 +17,9 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
   gem.add_dependency "azure", [">= 0.7.1", "<= 0.7.7"]
-  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_dependency "uuidtools", ">= 2.1.5"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", ">= 1.0.3"

data/lib/fluent/plugin/out_azurestorage.rb

@@ -1,16 +1,18 @@
-module Fluent
-  require 'fluent/mixin/config_placeholders'
-
-  class AzureStorageOutput < Fluent::TimeSlicedOutput
+require 'azure'
+require 'fluent/plugin/upload_service'
+require 'zlib'
+require 'time'
+require 'tempfile'
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class AzureStorageOutput < Fluent::Plugin::Output
     Fluent::Plugin.register_output('azurestorage', self)
 
+    helpers :compat_parameters, :formatter, :inject
+
     def initialize
       super
-      require 'azure'
-      require 'fluent/plugin/upload_service'
-      require 'zlib'
-      require 'time'
-      require 'tempfile'
 
       @compressor = nil
     end
@@ -26,27 +28,32 @@ module Fluent
     config_param :format, :string, :default => "out_file"
     config_param :command_parameter, :string, :default => nil
 
-    attr_reader :bs
+    DEFAULT_FORMAT_TYPE = "out_file"
 
-    include Fluent::Mixin::ConfigPlaceholders
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
 
-    def placeholders
-      [:percent]
+    config_section :buffer do
+      config_set_default :chunk_keys, ['time']
+      config_set_default :timekey, (60 * 60 * 24)
     end
 
+    attr_reader :bs
+
     def configure(conf)
+      compat_parameters_convert(conf, :buffer, :formatter, :inject)
       super
 
       begin
         @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
       rescue => e
-        $log.warn "#{@store_as} not found. Use 'text' instead"
+        log.warn "#{@store_as} not found. Use 'text' instead"
         @compressor = TextCompressor.new
       end
       @compressor.configure(conf)
 
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
 
       if @localtime
         @path_slicer = Proc.new {|path|
@@ -70,6 +77,13 @@ module Fluent
                         else
                           'blob'
                       end
+      # For backward compatibility
+      # TODO: Remove time_slice_format when end of support compat_parameters
+      @configured_time_slice_format = conf['time_slice_format']
+    end
+
+    def multi_workers_ready?
+      true
     end
 
     def start
@@ -88,25 +102,32 @@ module Fluent
     end
 
     def format(tag, time, record)
-      @formatter.format(tag, time, record)
+      r = inject_values_to_record(tag, time, record)
+      @formatter.format(tag, time, r)
     end
 
     def write(chunk)
       i = 0
+      metadata = chunk.metadata
       previous_path = nil
+      time_slice_format = @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey'])
+      time_slice = if metadata.timekey.nil?
+                     ''.freeze
+                   else
+                     Time.at(metadata.timekey).utc.strftime(time_slice_format)
+                   end
 
       begin
         path = @path_slicer.call(@path)
         values_for_object_key = {
-          "path" => path,
-          "time_slice" => chunk.key,
-          "file_extension" => @compressor.ext,
-          "index" => i,
-          "uuid_flush" => uuid_random
-        }
-        storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
-          values_for_object_key[expr[2...expr.size-1]]
+          "%{path}" => path,
+          "%{time_slice}" => time_slice,
+          "%{file_extension}" => @compressor.ext,
+          "%{index}" => i,
+          "%{uuid_flush}" => uuid_random
         }
+        storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
+        storage_path = extract_placeholders(storage_path, metadata)
         if (i > 0) && (storage_path == previous_path)
           raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
         end
@@ -140,8 +161,24 @@ module Fluent
       end
     end
 
+    def uuid_random
+      require 'uuidtools'
+      ::UUIDTools::UUID.random_create.to_s
+    end
+
+    # This is stolen from Fluentd
+    def timekey_to_timeformat(timekey)
+      case timekey
+      when nil          then ''
+      when 0...60       then '%Y%m%d%H%M%S' # 60 exclusive
+      when 60...3600    then '%Y%m%d%H%M'
+      when 3600...86400 then '%Y%m%d%H'
+      else                   '%Y%m%d'
+      end
+    end
+
     class Compressor
-      include Configurable
+      include Fluent::Configurable
 
       def initialize(opts = {})
         super()
@@ -220,7 +257,7 @@ module Fluent
       end
     end
 
-    COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
+    COMPRESSOR_REGISTRY = Fluent::Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
     {
         'gzip' => GzipCompressor,
         'json' => JsonCompressor,

data/test/test_out_azurestorage.rb

@@ -1,10 +1,14 @@
 require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
 require 'fluent/plugin/out_azurestorage'
 
 require 'test/unit/rr'
 require 'zlib'
 require 'fileutils'
 
+include Fluent::Test::Helpers
+
 class AzureStorageOutputTest < Test::Unit::TestCase
   def setup
     require 'azure'
@@ -21,9 +25,16 @@ class AzureStorageOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageOutput) do
+      # for testing.
+      def contents
+        @emit_streams
+      end
+
       def write(chunk)
-        chunk.read
+        @emit_streams = []
+        event = chunk.read
+        @emit_streams << event
       end
 
       private
@@ -64,11 +75,9 @@ class AzureStorageOutputTest < Test::Unit::TestCase
     conf = CONFIG.clone
     conf << "\nstore_as lzo\n"
     d = create_driver(conf)
-    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
-    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
-  rescue => e
-    # TODO: replace code with disable lzop command
-    assert(e.is_a?(Fluent::ConfigError))
+    # Fallback to text/plain.
+    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
   end
 
   def test_path_slicing
@@ -93,109 +102,118 @@ class AzureStorageOutputTest < Test::Unit::TestCase
   def test_format
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.run
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n], formatted[0]
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n], formatted[1]
   end
 
   def test_format_included_tag_and_time
     config = [CONFIG, 'include_tag_key true', 'include_time_key true'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-
-    d.run
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n], formatted[0]
+    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n], d.formatted[1]
   end
 
   def test_format_with_format_ltsv
     config = [CONFIG, 'format ltsv'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1, "b"=>1}, time)
-    d.emit({"a"=>2, "b"=>2}, time)
-
-    d.expect_format %[a:1\tb:1\n]
-    d.expect_format %[a:2\tb:2\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1, "b"=>1})
+      d.feed(time, {"a"=>2, "b"=>2})
+    end
+    formatted = d.formatted
 
-    d.run
+    assert_equal %[a:1\tb:1\n], formatted[0]
+    assert_equal %[a:2\tb:2\n], formatted[1]
   end
 
   def test_format_with_format_json
     config = [CONFIG, 'format json'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1}\n]
-    d.expect_format %[{"a":2}\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.run
+    assert_equal %[{"a":1}\n], formatted[0]
+    assert_equal %[{"a":2}\n], formatted[1]
   end
 
   def test_format_with_format_json_included_tag
     config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.expect_format %[{"a":1,"tag":"test"}\n]
-    d.expect_format %[{"a":2,"tag":"test"}\n]
-
-    d.run
+    assert_equal %[{"a":1,"tag":"test"}\n], formatted[0]
+    assert_equal %[{"a":2,"tag":"test"}\n], formatted[1]
   end
 
   def test_format_with_format_json_included_time
     config = [CONFIG, 'format json', 'include_time_key true'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.run
+    assert_equal %[{"a":1,"time":"2011-01-02T13:14:15Z"}\n], formatted[0]
+    assert_equal %[{"a":2,"time":"2011-01-02T13:14:15Z"}\n], formatted[1]
   end
 
   def test_format_with_format_json_included_tag_and_time
     config = [CONFIG, 'format json', 'include_tag_key true', 'include_time_key true'].join("\n")
     d = create_driver(config)
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
-    d.expect_format %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n]
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
+    formatted = d.formatted
 
-    d.run
+    assert_equal %[{"a":1,"tag":"test","time":"2011-01-02T13:14:15Z"}\n], formatted[0]
+    assert_equal %[{"a":2,"tag":"test","time":"2011-01-02T13:14:15Z"}\n], formatted[1]
   end
 
   def test_chunk_to_write
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
+    time = event_time("2011-01-02 13:14:15 UTC")
+    d.run(default_tag: "test") do
+      d.feed(time, {"a"=>1})
+      d.feed(time, {"a"=>2})
+    end
 
-    # AzureStorageOutputTest#write returns chunk.read
-    data = d.run
+    # Stubbed #write and #emit_streams returns chunk.read result.
+    data = d.instance.contents
 
     assert_equal [%[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
                  %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]],
@@ -216,7 +234,7 @@ class AzureStorageOutputTest < Test::Unit::TestCase
   ]
 
   def create_time_sliced_driver(conf = CONFIG_TIME_SLICE)
-    d = Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::AzureStorageOutput) do
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageOutput) do
     end.configure(conf)
     d
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.1.0
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-23 00:00:00.000000000 Z
+date: 2017-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -51,19 +51,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.7
 - !ruby/object:Gem::Dependency
-  name: fluent-mixin-config-placeholders
+  name: uuidtools
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 2.1.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 2.1.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement