fluent-plugin-azure-storage-append-blob 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 86a7a8f0c9164564d064640c1db1ccf3f97a3d0fb99fb6c14c65cdc0a104619c
+  data.tar.gz: c144705806010e9e6feb1d7bd84070156f413139a80ffdebad9325c639112329
+SHA512:
+  metadata.gz: e8ff1cf315f5f330f1d202db8cd0ca74ba77f41cdf11f961926f97eb9be8a3ecd963cba06ad7a8bad20fc4151e454f396ccc2c5b1568251127e99e1e4b2416ab
+  data.tar.gz: c27e7414feec0113b1d3b7bcc613c0a351e3eaac60e37bca3f8f04b9c66248f92c7b8f2a81ea71de375f62008d5f6d40cf29937829dd3a2e4c45d3b27fa1c6f1

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

data/README.md

@@ -0,0 +1,138 @@
+# fluent-plugin-azure-storage-append-blob
+
+[Fluentd](https://fluentd.org/) out plugin to do something.
+
+Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install fluent-plugin-azure-storage-append-blob
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "fluent-plugin-azure-storage-append-blob"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+```
+<match pattern>
+  type azure-storage-append-blob
+
+  azure_storage_account    <your azure storage account>
+  azure_storage_access_key <your azure storage access key>
+  azure_container          <your azure storage container>
+  auto_create_container    true
+  path logs/
+  azure_blob_name_format   %{path}%{time_slice}_%{index}.log
+  time_slice_format        %Y%m%d-%H
+  # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
+  # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/azurestorageappendblob
+    timekey 120 # 2 minutes
+    timekey_wait 60
+    timekey_use_utc true # use utc
+  </buffer>
+</match>
+```
+
+### azure_storage_account (Required)
+
+Your Azure Storage Account Name. This can be retrieved from Azure Management potal.
+
+### azure_storage_access_key (Required)
+
+Your Azure Storage Access Key(Primary or Secondary). This also can be retrieved from Azure Management potal.
+
+### azure_container (Required)
+
+Azure Storage Container name
+
+### auto_create_container
+
+This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
+The default value is `true`
+
+### azure_object_key_format
+
+The format of Azure Storage object keys. You can use several built-in variables:
+
+- %{path}
+- %{time_slice}
+- %{index}
+
+to decide keys dynamically.
+
+%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
+%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
+%{index} is used only if your blob exceed Azure 50000 blocks limit per blob to prevent data loss. Its not required to use this parameter.
+
+The default format is "%{path}%{time_slice}-%{index}.log".
+
+For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
+
+```
+"logs/20130111-22-0.log"
+"logs/20130111-23-0.log"
+"logs/20130112-00-0.log"
+```
+
+With the configuration:
+
+```
+azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
+path log
+time_slice_format %Y%m%d-%H
+```
+
+You get:
+
+```
+"log/events/ts=20130111-22/events.log"
+"log/events/ts=20130111-23/events.log"
+"log/events/ts=20130112-00/events.log"
+```
+
+The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
+
+```
+azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
+```
+
+### time_slice_format
+
+Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
+
+### Run tests
+    $ gem install bundler
+    $ bundle install
+    $ bundle exec rake test
+
+# Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

data/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-azure-storage-append-blob.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-azure-storage-append-blob"
+  spec.version = "0.1.0"
+  spec.authors = ["Microsoft"]
+  spec.email   = [""]
+
+  spec.summary       = "Azure Storage Append Blob output plugin for Fluentd event collector"
+  spec.description   = "Fluentd plugin to upload logs to Azure Storage append blobs."
+  spec.homepage      = "https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob"
+  spec.license       = "MIT"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+  spec.add_runtime_dependency "azure-storage-blob", "~> 1.0"
+end

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -0,0 +1,175 @@
+#---------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License. See License.txt in the project root for license information.
+#--------------------------------------------------------------------------------------------*/
+
+require 'fluent/plugin/output'
+require 'azure/storage/blob'
+require 'time'
+require 'tempfile'
+
+module Fluent
+  module Plugin
+    class AzureStorageAppendBlobOut < Fluent::Plugin::Output
+      Fluent::Plugin.register_output("azure-storage-append-blob", self)
+
+      helpers :formatter, :inject
+
+      DEFAULT_FORMAT_TYPE = "out_file"
+  
+      config_param :path, :string, :default => ""
+      config_param :azure_storage_account, :string, :default => nil
+      config_param :azure_storage_access_key, :string, :default => nil, :secret => true
+      config_param :azure_container, :string, :default => nil
+      config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}-%{index}.log"
+      config_param :auto_create_container, :bool, :default => true
+      config_param :format, :string, :default => DEFAULT_FORMAT_TYPE
+      config_param :time_slice_format, :string, :default => '%Y%m%d'
+  
+      DEFAULT_FORMAT_TYPE = "out_file"
+      AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
+  
+      config_section :format do
+        config_set_default :@type, DEFAULT_FORMAT_TYPE
+      end
+  
+      config_section :buffer do
+        config_set_default :chunk_keys, ['time']
+        config_set_default :timekey, (60 * 60 * 24)
+      end
+  
+      attr_reader :bs
+  
+      def configure(conf)
+        super
+  
+        @formatter = formatter_create
+  
+        if @localtime
+          @path_slicer = Proc.new {|path|
+            Time.now.strftime(path)
+          }
+        else
+          @path_slicer = Proc.new {|path|
+            Time.now.utc.strftime(path)
+          }
+        end
+  
+        if @azure_container.nil?
+          raise ConfigError, 'azure_container is needed'
+        end
+      end
+  
+      def multi_workers_ready?
+        true
+      end
+  
+      def start
+        super
+  
+        @bs = Azure::Storage::Blob::BlobService.create(storage_account_name: @azure_storage_account, storage_access_key: @azure_storage_access_key)
+  
+        ensure_container
+
+        @azure_storage_path = ''
+        @last_azure_storage_path = ''
+        @current_index = 0
+      end
+  
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+  
+      def write(chunk)
+        metadata = chunk.metadata
+        tmp = Tempfile.new("azure-")
+        begin
+          chunk.write_to(tmp)
+          tmp.close
+
+          generate_log_name(metadata, @current_index)
+          if @last_azure_storage_path != @azure_storage_path
+            @current_index = 0
+            generate_log_name(metadata, @current_index)
+          end
+
+          content = File.open(tmp.path, 'rb') { |file| file.read }
+
+          append_blob(content)
+          @last_azure_storage_path = @azure_storage_path
+        ensure
+          tmp.unlink
+        end
+      end
+  
+      private
+      def ensure_container
+        if ! @bs.list_containers.find { |c| c.name == @azure_container }
+          if @auto_create_container
+            @bs.create_container(@azure_container)
+          else
+            raise "The specified container does not exist: container = #{@azure_container}"
+          end
+        end
+      end
+
+      private
+      def generate_log_name(metadata, index)
+        time_slice = if metadata.timekey.nil?
+                       ''.freeze
+                     else
+                       Time.at(metadata.timekey).utc.strftime(@time_slice_format)
+                     end
+
+        path = @path_slicer.call(@path)
+        values_for_object_key = {
+          "%{path}" => path,
+          "%{time_slice}" => time_slice,
+          "%{index}" => index
+        }
+        storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
+        @azure_storage_path = extract_placeholders(storage_path, metadata)
+      end
+
+      private
+      def append_blob(content)
+        position = 0
+        log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
+        loop do
+          begin
+            size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+            log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
+            @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size])
+            position += size
+            break if position >= content.length
+          rescue Azure::Core::Http::HTTPError => ex
+            status_code = ex.status_code
+
+            if status_code == 409 # exceeds azure block limit
+              @current_index += 1
+              old_azure_storage_path = @azure_storage_path
+              generate_log_name(metadata, time_slice, @current_index)
+
+              # If index is not a part of format, rethrow exception.
+              if old_azure_storage_path == @azure_storage_path
+                log.warn "azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format."
+                raise
+              end
+
+              log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+              @bs.create_append_blob(@azure_container, @azure_storage_path)
+            elsif status_code == 404 # blob not found
+              log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+              @bs.create_append_blob(@azure_container, @azure_storage_path)
+            else
+              raise
+            end
+          end
+        end
+        log.debug "azure_storage_append_blob: append_blob.complete"
+      end
+
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -0,0 +1,66 @@
+require 'helper'
+require 'fluent/plugin/out_azure-storage-append-blob.rb'
+
+include Fluent::Test::Helpers
+
+class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    azure_storage_account test_storage_account
+    azure_storage_access_key MY_FAKE_SECRET
+    azure_container test_container
+    time_slice_format        %Y%m%d-%H
+    path log
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
+  end
+
+  sub_test_case 'test config' do
+    test 'config should reject with no azure container' do
+      assert_raise Fluent::ConfigError do
+        create_driver(%[
+          azure_storage_account test_storage_account
+          azure_storage_access_key MY_FAKE_SECRET
+          time_slice_format        %Y%m%d-%H
+          time_slice_wait          10m
+          path log
+        ])
+      end
+    end
+
+     test 'config should set instance variables' do
+      d = create_driver
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+    end
+  end
+
+  sub_test_case 'test path slicing' do
+    test 'test path_slicing' do
+      config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
+      d = create_driver(config)
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
+    end
+
+    test 'path slicing utc' do
+      config = CONFIG.clone.gsub(/path\slog/, "path log/%Y/%m/%d")
+      config << "\nutc\n"
+      d = create_driver(config)
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime("log/%Y/%m/%d")
+    end
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azure-storage-append-blob
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Microsoft
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: azure-storage-blob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Fluentd plugin to upload logs to Azure Storage append blobs.
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-azure-storage-append-blob.gemspec
+- lib/fluent/plugin/out_azure-storage-append-blob.rb
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb
+homepage: https://github.com/Microsoft/fluent-plugin-azure-storage-append-blob
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.8
+signing_key: 
+specification_version: 4
+summary: Azure Storage Append Blob output plugin for Fluentd event collector
+test_files:
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb