fluent-plugin-azure-storage-append-blob-lts 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 335e03eeafd0648547d44b98ce7bbbc3dd8a780dc03d3e39a17005a8305c0ba1
+  data.tar.gz: e1361ff0a6cc0d95eec72e2b330861de638afd17601a3ba2d6f81d0a18d7ee9a
+SHA512:
+  metadata.gz: b2c40956a6eb51ea8a0d19058527fe8c470d48dbf59130671322c99c073353ffb4b9a0f931afcf7707dbc3deda7d44974174214e3045f7d566da876b7f434e93
+  data.tar.gz: 31258dfaf599e2bcfdda8c268aef6295b9720407c93a8b8997376ef192b7024fe15639da751be765c43e562738ad7f50daf479055f9668ef3cd7748e41173dd2

data/.github/workflows/publish.yaml

@@ -0,0 +1,31 @@
+name: Publish Ruby Gem
+
+on:
+  workflow_dispatch:
+    inputs:
+      otp:
+        description: 'One Time Password'
+        required: true
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem --otp ${{ github.event.inputs.otp }}
+      env:
+        RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/test.yaml

@@ -0,0 +1,23 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Style/FrozenStringLiteralComment:
+  Enabled: false

data/Dockerfile

@@ -0,0 +1,37 @@
+FROM ruby:latest
+
+WORKDIR /plugin
+
+ADD . /plugin
+
+RUN gem install bundler && \
+    gem install fluentd --no-doc && \
+    fluent-gem build fluent-plugin-azure-storage-append-blob-lts.gemspec && \
+    fluent-gem install fluent-plugin-azure-storage-append-blob-lts-*.gem
+
+RUN echo "<source>\n\
+  @type sample\n\
+  sample {\"hello\":\"world\"}\n\
+  tag pattern\n\
+</source>\n\
+<match pattern>\n\
+  @type azure-storage-append-blob\n\
+  azure_storage_account             \"#{ENV['STORAGE_ACCOUNT']}\"\n\
+  azure_storage_access_key          \"#{ENV['STORAGE_ACCESS_KEY']}\"\n\
+  azure_storage_connection_string   \"#{ENV['STORAGE_CONNECTION_STRING']}\"\n\
+  azure_storage_sas_token           \"#{ENV['STORAGE_SAS_TOKEN']}\"\n\
+  azure_container                   fluentd\n\
+  auto_create_container             true\n\
+  path                              logs/\n\
+  azure_object_key_format           %{path}%{time_slice}_%{index}.log\n\
+  time_slice_format                 %Y%m%d-%H\n\
+  <buffer tag,time>\n\
+    @type file\n\
+    path /var/log/fluent/azurestorageappendblob\n\
+    timekey 120 # 2 minutes\n\
+    timekey_wait 60\n\
+    timekey_use_utc true # use utc\n\
+  </buffer>\n\
+</match>" > /plugin/fluent.conf
+
+ENTRYPOINT ["fluentd", "-c", "fluent.conf"]

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

data/README.md

@@ -0,0 +1,170 @@
+# fluent-plugin-azure-storage-append-blob-lts
+
+Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
+This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
+
+## Installation
+
+### RubyGems
+
+    gem install fluent-plugin-azure-storage-append-blob-lts
+
+### Bundler
+
+Add following line to your Gemfile:
+
+    gem "fluent-plugin-azure-storage-append-blob-lts"
+
+And then execute:
+
+    bundle
+
+## Configuration
+
+    <match pattern>
+      type azure-storage-append-blob
+
+      azure_storage_account             <your azure storage account>
+      azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
+      azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
+      azure_storage_sas_token           <your azure storage sas token> # leave empty to use MSI
+      azure_imds_api_version            <Azure Instance Metadata Service API Version> # only used for MSI
+      azure_token_refresh_interval      <refresh interval in min> # only used for MSI
+      azure_container                   <your azure storage container>
+      auto_create_container             true
+      path                              logs/
+      azure_object_key_format           %{path}%{time_slice}_%{index}.log
+      time_slice_format                 %Y%m%d-%H
+      # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
+      # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
+      <buffer tag,time>
+        @type file
+        path /var/log/fluent/azurestorageappendblob
+        timekey 120 # 2 minutes
+        timekey_wait 60
+        timekey_use_utc true # use utc
+      </buffer>
+    </match>
+
+### `azure_storage_account` (Required)
+
+Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
+
+### `azure_storage_access_key` or `azure_storage_sas_token` or `azure_storage_connection_string` (Any required or all empty to use MSI)
+
+Your Azure Storage Access Key (Primary or Secondary) or shared access signature (SAS) token or full connection string.
+This also can be retrieved from Azure Management portal.
+
+If all are empty, the plugin will use the local Managed Identity endpoint to obtain a token for the target storage account.
+
+### `azure_imds_api_version` (Optional, only for MSI)
+
+Default: 2019-08-15
+
+The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
+
+See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
+
+### `azure_token_refresh_interval` (Optional, only for MSI)
+
+Default: 60 (1 hour)
+
+When using MSI, the initial access token needs to be refreshed periodically.
+
+### `azure_container` (Required)
+
+Azure Storage Container name
+
+### `auto_create_container`
+
+This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
+The default value is `true`
+
+### `azure_object_key_format`
+
+The format of Azure Storage object keys. You can use several built-in variables:
+
+- %{path}
+- %{time_slice}
+- %{index}
+
+to decide keys dynamically.
+
+%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
+%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
+%{index} is used only if your blob exceed Azure 50000 blocks limit per blob to prevent data loss. Its not required to use this parameter.
+
+The default format is "%{path}%{time_slice}-%{index}.log".
+
+For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
+
+    "logs/20130111-22-0.log"
+    "logs/20130111-23-0.log"
+    "logs/20130112-00-0.log"
+
+With the configuration:
+
+    azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
+    path log
+    time_slice_format %Y%m%d-%H
+
+You get:
+
+    "log/events/ts=20130111-22/events.log"
+    "log/events/ts=20130111-23/events.log"
+    "log/events/ts=20130112-00/events.log"
+
+The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
+
+    azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
+
+### `time_slice_format`
+
+Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
+
+### Run tests
+
+    gem install bundler
+    bundle install
+    bundle exec rake test
+
+
+### Test Fluentd
+
+1. Create Storage Account and VM with enabled MSI
+2. Setup Docker ang Git
+3. SSH into VM
+4. Download this repo
+   ```
+   git clone https://github.com/microsoft/fluent-plugin-azure-storage-append-blob-lts.git
+   cd fluent-plugin-azure-storage-append-blob-lts
+   ```
+5. Build Docker image
+   `docker build -t fluent .`
+6. Run Docker image with different set of parameters:
+
+    1. `STORAGE_ACCOUNT`: required, name of your storage account
+    2. `STORAGE_ACCESS_KEY`: storage account access key
+    3. `STORAGE_SAS_TOKEN`: storage sas token with enough permissions for the plugin
+
+    You need to specify `STORAGE_ACCOUNT` and one of auth ways. If you run it from VM with MSI,
+    just `STORAGE_ACCOUNT` is required. Keep in mind, there is no way to refresh MSI Token, so
+    ensure you setup proper permissions first.
+
+    ```bash
+    docker run -it -e STORAGE_ACCOUNT=<storage> -e STORAGE_ACCESS_KEY=<key> fluent
+    ```
+
+## Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com).
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push('lib', 'test')
+  t.test_files = FileList['test/**/test_*.rb']
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -0,0 +1,29 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
+  spec.version = '0.3.0'
+  spec.authors = ['Jonas-Taha El Sesiy']
+  spec.email = ['']
+
+  spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
+  spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob'
+  spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
+  spec.license = 'MIT'
+
+  test_files, files = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files = files
+  spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files = test_files
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.6'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
+end

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -0,0 +1,235 @@
+#---------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License. See License.txt in the project root for license information.
+#--------------------------------------------------------------------------------------------*/
+
+require 'azure/storage/common'
+require 'azure/storage/blob'
+require 'faraday'
+require 'fluent/plugin/output'
+require 'json'
+
+module Fluent
+  module Plugin
+    class AzureStorageAppendBlobOut < Fluent::Plugin::Output
+      Fluent::Plugin.register_output('azure-storage-append-blob', self)
+
+      helpers :formatter, :inject
+
+      DEFAULT_FORMAT_TYPE = 'out_file'.freeze
+      AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
+
+      config_param :path, :string, default: ''
+      config_param :azure_storage_account, :string, default: nil
+      config_param :azure_storage_access_key, :string, default: nil, secret: true
+      config_param :azure_storage_sas_token, :string, default: nil, secret: true
+      config_param :azure_storage_connection_string, :string, default: nil, secret: true
+      config_param :azure_container, :string, default: nil
+      config_param :azure_imds_api_version, :string, default: '2019-08-15'
+      config_param :azure_token_refresh_interval, :integer, default: 60
+      config_param :use_msi, :bool, default: false
+      config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
+      config_param :auto_create_container, :bool, default: true
+      config_param :format, :string, default: DEFAULT_FORMAT_TYPE
+      config_param :time_slice_format, :string, default: '%Y%m%d'
+      config_param :localtime, :bool, default: false
+
+      config_section :format do
+        config_set_default :@type, DEFAULT_FORMAT_TYPE
+      end
+
+      config_section :buffer do
+        config_set_default :chunk_keys, ['time']
+        config_set_default :timekey, (60 * 60 * 24)
+      end
+
+      attr_reader :bs
+
+      def configure(conf)
+        super
+
+        @formatter = formatter_create
+
+        @path_slicer = if @localtime
+                         proc do |path|
+                           Time.now.strftime(path)
+                         end
+                       else
+                         proc do |path|
+                           Time.now.utc.strftime(path)
+                         end
+                       end
+
+        raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
+
+        raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
+
+        if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
+           (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
+           (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
+          log.info 'Using MSI since neither azure_storage_access_key nor azure_storage_sas_token was provided.'
+          @use_msi = true
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get_access_token
+        access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
+                                         "api-version=#{@azure_imds_api_version}" \
+                                         '&resource=https://storage.azure.com/',
+                                         headers: { 'Metadata' => 'true' })
+                                    .get
+                                    .body
+        JSON.parse(access_key_request)['access_token']
+      end
+
+      def start
+        super
+        if @use_msi
+          token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
+          token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
+          @bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
+
+          refresh_interval = @azure_token_refresh_interval * 60
+          cancelled = false
+          renew_token = Thread.new do
+            Thread.stop
+            until cancelled
+              sleep(refresh_interval)
+
+              token_credential.renew_token get_access_token
+            end
+          end
+          sleep 0.1 while renew_token.status != 'sleep'
+          renew_token.run
+        elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
+          @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
+        else
+          @bs_params = { storage_account_name: @azure_storage_account }
+
+          if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
+            @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
+          elsif !@azure_storage_sas_token.nil? && !@azure_storage_sas_token.empty?
+            @bs_params.merge!({ storage_sas_token: @azure_storage_sas_token })
+          end
+
+          @bs = Azure::Storage::Blob::BlobService.create(@bs_params)
+
+        end
+
+        ensure_container
+        @azure_storage_path = ''
+        @last_azure_storage_path = ''
+        @current_index = 0
+      end
+
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+
+      def write(chunk)
+        metadata = chunk.metadata
+        tmp = Tempfile.new('azure-')
+        begin
+          chunk.write_to(tmp)
+
+          generate_log_name(metadata, @current_index)
+          if @last_azure_storage_path != @azure_storage_path
+            @current_index = 0
+            generate_log_name(metadata, @current_index)
+          end
+
+          content = File.open(tmp.path, 'rb', &:read)
+
+          append_blob(content, metadata)
+          @last_azure_storage_path = @azure_storage_path
+        ensure
+          tmp.close(true) rescue nil
+        end
+      end
+
+      def container_exists?(container)
+        begin
+          @bs.get_container_properties(container)
+        rescue Azure::Core::Http::HTTPError => e
+          if e.status_code == 404 # container does not exist
+            return false
+          else
+            raise
+          end
+        end
+        true
+      end
+
+      private
+
+      def ensure_container
+        unless container_exists? @azure_container
+          if @auto_create_container
+            @bs.create_container(@azure_container)
+          else
+            raise "The specified container does not exist: container = #{@azure_container}"
+          end
+        end
+      end
+
+      def generate_log_name(metadata, index)
+        time_slice = if metadata.timekey.nil?
+                       ''.freeze
+                     else
+                       Time.at(metadata.timekey).utc.strftime(@time_slice_format)
+                     end
+
+        path = @path_slicer.call(@path)
+        values_for_object_key = {
+          '%{path}' => path,
+          '%{time_slice}' => time_slice,
+          '%{index}' => index
+        }
+        storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
+        @azure_storage_path = extract_placeholders(storage_path, metadata)
+      end
+
+      def append_blob(content, metadata)
+        position = 0
+        log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
+        loop do
+          begin
+            size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+            log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
+            @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1])
+            position += size
+            break if position >= content.length
+          rescue Azure::Core::Http::HTTPError => e
+            status_code = e.status_code
+
+            if status_code == 409 # exceeds azure block limit
+              @current_index += 1
+              old_azure_storage_path = @azure_storage_path
+              generate_log_name(metadata, @current_index)
+
+              # If index is not a part of format, rethrow exception.
+              if old_azure_storage_path == @azure_storage_path
+                log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
+                raise
+              end
+
+              log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+              @bs.create_append_blob(@azure_container, @azure_storage_path)
+            elsif status_code == 404 # blob not found
+              log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+              @bs.create_append_blob(@azure_container, @azure_storage_path)
+            else
+              raise
+            end
+          end
+        end
+        log.debug 'azure_storage_append_blob: append_blob.complete'
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -0,0 +1,184 @@
+require 'helper'
+require 'fluent/plugin/out_azure-storage-append-blob.rb'
+require 'azure/core/http/http_response'
+require 'azure/core/http/http_error'
+
+include Fluent::Test::Helpers
+
+class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  CONFIG = %(
+    azure_storage_account test_storage_account
+    azure_storage_access_key MY_FAKE_SECRET
+    azure_container test_container
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
+  MSI_CONFIG = %(
+    azure_storage_account test_storage_account
+    azure_container test_container
+    azure_imds_api_version 1970-01-01
+    azure_token_refresh_interval 120
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
+  def create_driver(conf: CONFIG, service: nil)
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
+    d.instance.instance_variable_set(:@bs, service)
+    d.instance.instance_variable_set(:@azure_storage_path, 'storage_path')
+    d
+  end
+
+  sub_test_case 'test config' do
+    test 'config should reject with no azure container' do
+      assert_raise Fluent::ConfigError do
+        create_driver conf: %(
+          azure_storage_account test_storage_account
+          azure_storage_access_key MY_FAKE_SECRET
+          time_slice_format        %Y%m%d-%H
+          time_slice_wait          10m
+          path log
+        )
+      end
+    end
+
+    test 'config with access key should set instance variables' do
+      d = create_driver
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+    end
+
+    test 'config with managed identity enabled should set instance variables' do
+      d = create_driver conf: MSI_CONFIG
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.use_msi
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+      assert_equal 120, d.instance.azure_token_refresh_interval
+      assert_equal '1970-01-01', d.instance.azure_imds_api_version
+    end
+  end
+
+  sub_test_case 'test path slicing' do
+    test 'test path_slicing' do
+      config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
+      d = create_driver conf: config
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
+    end
+
+    test 'path slicing utc' do
+      config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
+      config << "\nutc\n"
+      d = create_driver conf: config
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
+    end
+  end
+
+  # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
+  # a response object when it is created.
+  class FakeResponse
+    def initialize(status=404)
+      @status = status
+      @body = "body"
+      @headers = {}
+    end
+
+    attr_reader :status
+    attr_reader :body
+    attr_reader :headers
+  end
+
+  # This class is used to test plugin functions which interact with the blob service
+  class FakeBlobService
+    def initialize(status)
+      @response = Azure::Core::Http::HttpResponse.new(FakeResponse.new(status))
+      @blocks = []
+    end
+    attr_reader :blocks
+
+    def append_blob_block(container, path, data)
+      @blocks.append(data)
+    end
+
+    def get_container_properties(container)
+      unless @response.status_code == 200
+        raise Azure::Core::Http::HTTPError.new(@response)
+      end
+    end
+  end
+
+  sub_test_case 'test container_exists' do
+    test 'container 404 returns false' do
+      d = create_driver service: FakeBlobService.new(404)
+      assert_false d.instance.container_exists? "anything"
+    end
+
+    test 'existing container returns true' do
+      d = create_driver service: FakeBlobService.new(200)
+      assert_true d.instance.container_exists? "anything"
+    end
+
+    test 'unexpected exception raises' do
+      d = create_driver service: FakeBlobService.new(500)
+      assert_raise_kind_of Azure::Core::Http::HTTPError do
+        d.instance.container_exists? "anything"
+      end
+    end
+  end
+
+  # Override the block size limit so that mocked requests do not require huge buffers
+  class Fluent::Plugin::AzureStorageAppendBlobOut
+    AZURE_BLOCK_SIZE_LIMIT = 10
+  end
+
+  sub_test_case 'test append blob buffering' do
+    def fake_appended_blocks(content)
+      # run the append on the fake blob service, return a list of append request buffers
+      svc = FakeBlobService.new(200)
+      d = create_driver service: svc
+      d.instance.send(:append_blob, content, nil)
+      svc.blocks
+    end
+
+    test 'short buffer appends once' do
+      content = '123456789'
+      blocks = fake_appended_blocks content
+      assert_equal [content], blocks
+    end
+
+    test 'single character appends once' do
+      content = '1'
+      blocks = fake_appended_blocks content
+      assert_equal [content], blocks
+    end
+
+    test 'empty appends once' do
+      content = ''
+      blocks = fake_appended_blocks content
+      assert_equal [''], blocks
+    end
+
+    test 'long buffer appends multiple times' do
+      limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
+      buf_1 = 'a' * limit
+      buf_2 = 'a' * 3
+      blocks = fake_appended_blocks buf_1 + buf_2
+      assert_equal [buf_1, buf_2], blocks
+    end
+  end
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azure-storage-append-blob-lts
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Jonas-Taha El Sesiy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: azure-storage-blob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
+  https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/publish.yaml"
+- ".github/workflows/test.yaml"
+- ".gitignore"
+- ".rubocop.yml"
+- Dockerfile
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-azure-storage-append-blob-lts.gemspec
+- lib/fluent/plugin/out_azure-storage-append-blob.rb
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb
+homepage: https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Azure Storage Append Blob output plugin for Fluentd event collector
+test_files:
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb