fluent-plugin-azure-storage-append-blob-lts-azurestack 0.6.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 539fe91fe557c8022c49e79c26022a43ebce8a5e43c0e2932881ad4a63f3ac68
+  data.tar.gz: 624604c25bc0a3df095329a8657be820cdbe57f9543392b261e99d3e75bb25b4
+SHA512:
+  metadata.gz: dfe527fe5a38f26a0c6fe5c7ae581e96422ae0f4e3dfb482409e82689b17eb5ed13dca24244f770744b11d7515dc8409eb57f78c14964978e9d9b8327734209d
+  data.tar.gz: bc6527c9b5434598249b0f8df3b7f2859bcab6889d32e20407716ec7d70150c8d5855761a43c70c15d91de460b905e27d707a21320595b2c59152f424a7c31e5

data/.github/workflows/publish.yaml

@@ -0,0 +1,29 @@
+name: Publish Ruby Gem
+
+on:
+  workflow_dispatch:
+    inputs:
+      otp:
+        description: 'One Time Password'
+        required: true
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem --otp ${{ github.event.inputs.otp }}
+      env:
+        RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Style/FrozenStringLiteralComment:
+  Enabled: false

data/Dockerfile

@@ -0,0 +1,37 @@
+FROM ruby:2.7
+
+WORKDIR /plugin
+
+COPY . /plugin
+
+RUN gem install bundler && \
+    gem install fluentd --no-doc && \
+    fluent-gem build fluent-plugin-azure-storage-append-blob-lts.gemspec && \
+    fluent-gem install fluent-plugin-azure-storage-append-blob-lts-*.gem
+
+RUN echo "<source>\n\
+  @type sample\n\
+  sample {\"hello\":\"world\"}\n\
+  tag pattern\n\
+</source>\n\
+<match pattern>\n\
+  @type azure-storage-append-blob\n\
+  azure_storage_account             \"#{ENV['STORAGE_ACCOUNT']}\"\n\
+  azure_storage_access_key          \"#{ENV['STORAGE_ACCESS_KEY']}\"\n\
+  azure_storage_connection_string   \"#{ENV['STORAGE_CONNECTION_STRING']}\"\n\
+  azure_storage_sas_token           \"#{ENV['STORAGE_SAS_TOKEN']}\"\n\
+  azure_container                   fluentd\n\
+  auto_create_container             true\n\
+  path                              logs/\n\
+  azure_object_key_format           %{path}%{time_slice}_%{index}.log\n\
+  time_slice_format                 %Y%m%d-%H\n\
+  <buffer tag,time>\n\
+    @type file\n\
+    path /var/log/fluent/azurestorageappendblob\n\
+    timekey 120 # 2 minutes\n\
+    timekey_wait 60\n\
+    timekey_use_utc true # use utc\n\
+  </buffer>\n\
+</match>" > /plugin/fluent.conf
+
+ENTRYPOINT ["fluentd", "-c", "fluent.conf"]

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

data/README.md

@@ -0,0 +1,196 @@
+# fluent-plugin-azure-storage-append-blob-lts
+
+![Tests](https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts/workflows/Test/badge.svg?branch=master)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts.svg)](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts)
+[![Twitter](https://img.shields.io/badge/twitter-@elsesiy-blue.svg)](http://twitter.com/elsesiy)
+
+Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
+This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
+
+## Installation
+
+### RubyGems
+
+    gem install fluent-plugin-azure-storage-append-blob-lts
+
+### Bundler
+
+Add following line to your Gemfile:
+
+    gem "fluent-plugin-azure-storage-append-blob-lts"
+
+And then execute:
+
+    bundle
+
+## Configuration
+
+    <match pattern>
+      type azure-storage-append-blob
+
+      azure_cloud                       <azure cloud environment>
+      azure_storage_account             <your azure storage account>
+      azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
+      azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
+      azure_storage_sas_token           <your azure storage sas token> # leave empty to use MSI
+      azure_imds_api_version            <Azure Instance Metadata Service API Version> # only used for MSI
+      azure_token_refresh_interval      <refresh interval in min> # only used for MSI
+      azure_container                   <your azure storage container>
+      azure_msi_client_id               <Azure Managed Identity Client ID> # only used for MSI
+      auto_create_container             true
+      path                              logs/
+      azure_object_key_format           %{path}%{time_slice}_%{index}.log
+      time_slice_format                 %Y%m%d-%H
+      compute_checksums                 true
+      # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
+      # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
+      <buffer tag,time>
+        @type file
+        path /var/log/fluent/azurestorageappendblob
+        timekey 120 # 2 minutes
+        timekey_wait 60
+        timekey_use_utc true # use utc
+      </buffer>
+    </match>
+
+### `azure_cloud` (Optional)
+
+Default: `AZUREPUBLICCLOUD`
+
+Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
+
+### `azure_storage_account` (Required)
+
+Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
+
+### `azure_storage_access_key` or `azure_storage_sas_token` or `azure_storage_connection_string` (Any required or all empty to use MSI)
+
+Your Azure Storage Access Key (Primary or Secondary) or shared access signature (SAS) token or full connection string.
+This also can be retrieved from Azure Management portal.
+
+If all are empty, the plugin will use the local Managed Identity endpoint to obtain a token for the target storage account.
+
+### `azure_imds_api_version` (Optional, only for MSI)
+
+Default: `2019-08-15`
+
+The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
+
+See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
+
+### `azure_token_refresh_interval` (Optional, only for MSI)
+
+Default: `60` (1 hour)
+
+When using MSI, the initial access token needs to be refreshed periodically.
+
+### `azure_container` (Required)
+
+Azure Storage Container name
+
+### `azure_msi_client_id` (Optional, only for MSI)
+
+Azure Identity Client ID to use for accessing Azure Blob service.
+
+### `auto_create_container`
+
+Default: `true`
+
+This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
+
+### `azure_object_key_format`
+
+The format of Azure Storage object keys. You can use several built-in variables:
+
+- %{path}
+- %{time_slice}
+- %{index}
+
+to decide keys dynamically.
+
+%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
+%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
+%{index} is used only if your blob exceed Azure 50000 blocks limit per blob to prevent data loss. Its not required to use this parameter.
+
+The default format is "%{path}%{time_slice}-%{index}.log".
+
+For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
+
+    "logs/20130111-22-0.log"
+    "logs/20130111-23-0.log"
+    "logs/20130112-00-0.log"
+
+With the configuration:
+
+    azure_object_key_format %{path}/events/ts=%{time_slice}/events.log
+    path log
+    time_slice_format %Y%m%d-%H
+
+You get:
+
+    "log/events/ts=20130111-22/events.log"
+    "log/events/ts=20130111-23/events.log"
+    "log/events/ts=20130112-00/events.log"
+
+The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, etc. can be used in the `azure_object_key_format`. This is useful in preventing filename conflicts when writing from multiple servers.
+
+    azure_object_key_format %{path}/events/ts=%{time_slice}/events-%{hostname}.log
+
+### `time_slice_format`
+
+Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
+
+### `compute_checksums`
+
+Default: `true`
+
+Whether to compute MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
+
+You want to set it to `false` in FIPS-enabled environments.
+
+### Run tests
+
+    gem install bundler
+    bundle install
+    bundle exec rake test
+
+
+### Test Fluentd
+
+1. Create Storage Account and VM with enabled MSI
+2. Setup Docker ang Git
+3. SSH into VM
+4. Download this repo
+   ```
+   git clone https://github.com/microsoft/fluent-plugin-azure-storage-append-blob-lts.git
+   cd fluent-plugin-azure-storage-append-blob-lts
+   ```
+5. Build Docker image
+   `docker build -t fluent .`
+6. Run Docker image with different set of parameters:
+
+    1. `STORAGE_ACCOUNT`: required, name of your storage account
+    2. `STORAGE_ACCESS_KEY`: storage account access key
+    3. `STORAGE_SAS_TOKEN`: storage sas token with enough permissions for the plugin
+
+    You need to specify `STORAGE_ACCOUNT` and one of auth ways. If you run it from VM with MSI,
+    just `STORAGE_ACCOUNT` is required. Keep in mind, there is no way to refresh MSI Token, so
+    ensure you setup proper permissions first.
+
+    ```bash
+    docker run -it -e STORAGE_ACCOUNT=<storage> -e STORAGE_ACCESS_KEY=<key> fluent
+    ```
+
+## Contributing
+
+This project welcomes contributions and suggestions.  Most contributions require you to agree to a
+Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
+the rights to use your contribution. For details, visit [https://cla.microsoft.com](https://cla.microsoft.com).
+
+When you submit a pull request, a CLA-bot will automatically determine whether you need to provide
+a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions
+provided by the bot. You will only need to do this once across all repos using our CLA.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/).
+For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
+contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push('lib', 'test')
+  t.test_files = FileList['test/**/test_*.rb']
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -0,0 +1,29 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name = 'fluent-plugin-azure-storage-append-blob-lts-azurestack'
+  spec.version = '0.6.2'
+  spec.authors = ['Jiaxun Song']
+  spec.email = ['jiaxun.song@outlook.com']
+
+  spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
+  spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/microsoft/fluent-plugin-azure-storage-append-blob'
+  spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
+  spec.license = 'MIT'
+
+  test_files, files = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files = files
+  spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files = test_files
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.6'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'test-unit', '~> 3.0'
+  spec.add_runtime_dependency 'azure-storage-blob', '~> 2.0'
+  spec.add_runtime_dependency 'fluentd', ['>= 0.14.10', '< 2']
+end

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -0,0 +1,270 @@
+#---------------------------------------------------------------------------------------------
+#  Copyright (c) Microsoft Corporation. All rights reserved.
+#  Licensed under the MIT License. See License.txt in the project root for license information.
+#--------------------------------------------------------------------------------------------*/
+
+require 'azure/storage/common'
+require 'azure/storage/blob'
+require 'faraday'
+require 'fluent/plugin/output'
+require 'json'
+
+module Fluent
+  module Plugin
+    class AzureStorageAppendBlobOut < Output
+      Fluent::Plugin.register_output('azure-storage-append-blob', self)
+
+      helpers :formatter, :inject
+
+      DEFAULT_FORMAT_TYPE = 'out_file'.freeze
+      AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
+
+      def initialize
+        super
+        @use_msi = false
+
+        # Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
+        @storage_endpoint_mapping = {
+          'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
+          'AZUREGERMANCLOUD' => 'core.cloudapi.de',
+          'AZUREPUBLICCLOUD' => 'core.windows.net',
+          'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
+        }
+      end
+
+      config_param :path, :string, default: ''
+      config_param :azure_storage_account, :string, default: nil
+      config_param :azure_storage_access_key, :string, default: nil, secret: true
+      config_param :azure_storage_connection_string, :string, default: nil, secret: true
+      config_param :azure_storage_sas_token, :string, default: nil, secret: true
+      config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
+      config_param :azure_msi_client_id, :string, default: nil
+      config_param :azure_container, :string, default: nil
+      config_param :azure_imds_api_version, :string, default: '2019-08-15'
+      config_param :azure_token_refresh_interval, :integer, default: 60
+      config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
+      config_param :auto_create_container, :bool, default: true
+      config_param :compute_checksums, :bool, default: true
+      config_param :format, :string, default: DEFAULT_FORMAT_TYPE
+      config_param :time_slice_format, :string, default: '%Y%m%d'
+      config_param :localtime, :bool, default: false
+
+      config_section :format do
+        config_set_default :@type, DEFAULT_FORMAT_TYPE
+      end
+
+      config_section :buffer do
+        config_set_default :chunk_keys, ['time']
+        config_set_default :timekey, (60 * 60 * 24)
+      end
+
+      attr_reader :bs
+
+      def configure(conf)
+        super
+
+        @formatter = formatter_create
+
+        @path_slicer = if @localtime
+                         proc do |path|
+                           Time.now.strftime(path)
+                         end
+                       else
+                         proc do |path|
+                           Time.now.utc.strftime(path)
+                         end
+                       end
+
+        @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
+        if @azure_storage_dns_suffix.nil?
+          raise ConfigError 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD'
+        end
+
+        if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
+           (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
+           (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
+          log.info 'Using MSI since neither of azure_storage_access_key, azure_storage_sas_token, azure_storage_connection_string was provided.'
+          @use_msi = true
+        elsif @azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?
+          raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
+          raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
+        end
+
+        @blob_options = {}
+
+        if !@compute_checksums
+          @blob_options[:content_md5] = ''
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get_access_token
+        access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
+                                         "api-version=#{@azure_imds_api_version}" \
+                                         '&resource=https://storage.azure.com/' \
+                                         "#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
+                                         headers: { 'Metadata' => 'true' }).get
+
+        if access_key_request.status == 200
+          JSON.parse(access_key_request.body)['access_token']
+        else
+          raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
+        end
+      end
+
+      def start
+        super
+        if @use_msi
+          token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
+          token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
+          @bs = Azure::Storage::Blob::BlobService.new(
+            storage_account_name: @azure_storage_account,
+            storage_dns_suffix: @azure_storage_dns_suffix,
+            signer: token_signer
+          )
+
+          refresh_interval = @azure_token_refresh_interval * 60
+          cancelled = false
+          renew_token = Thread.new do
+            Thread.stop
+            until cancelled
+              sleep(refresh_interval)
+
+              token_credential.renew_token get_access_token
+            end
+          end
+          sleep 0.1 while renew_token.status != 'sleep'
+          renew_token.run
+        elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
+          @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
+        else
+          @bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
+
+          if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
+            @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
+          elsif !@azure_storage_sas_token.nil? && !@azure_storage_sas_token.empty?
+            @bs_params.merge!({ storage_sas_token: @azure_storage_sas_token })
+          end
+
+          @bs = Azure::Storage::Blob::BlobService.create(@bs_params)
+
+        end
+
+        ensure_container
+        @azure_storage_path = ''
+        @last_azure_storage_path = ''
+        @current_index = 0
+      end
+
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+
+      def write(chunk)
+        tmp = Tempfile.new('azure-')
+        begin
+          chunk.write_to(tmp)
+
+          generate_log_name(chunk, @current_index)
+          if @last_azure_storage_path != @azure_storage_path
+            @current_index = 0
+            generate_log_name(chunk, @current_index)
+          end
+
+          content = File.open(tmp.path, 'rb', &:read)
+
+          append_blob(content, chunk)
+          @last_azure_storage_path = @azure_storage_path
+        ensure
+          begin
+            tmp.close(true)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+
+      def container_exists?(container)
+        begin
+          @bs.get_container_properties(container)
+        rescue Azure::Core::Http::HTTPError => e
+          if e.status_code == 404 # container does not exist
+            return false
+          else
+            raise
+          end
+        end
+        true
+      end
+
+      private
+
+      def ensure_container
+        unless container_exists? @azure_container
+          if @auto_create_container
+            @bs.create_container(@azure_container)
+          else
+            raise "The specified container does not exist: container = #{@azure_container}"
+          end
+        end
+      end
+
+      def generate_log_name(chunk, index)
+        metadata = chunk.metadata
+        time_slice = if metadata.timekey.nil?
+                       ''.freeze
+                     else
+                       Time.at(metadata.timekey).utc.strftime(@time_slice_format)
+                     end
+
+        path = @path_slicer.call(@path)
+        values_for_object_key = {
+          '%{path}' => path,
+          '%{time_slice}' => time_slice,
+          '%{index}' => index
+        }
+        storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
+        @azure_storage_path = extract_placeholders(storage_path, chunk)
+      end
+
+      def append_blob(content, chunk)
+        position = 0
+        log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
+        loop do
+          size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+          log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
+          @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
+          position += size
+          break if position >= content.length
+        rescue Azure::Core::Http::HTTPError => e
+          status_code = e.status_code
+
+          if status_code == 409 # exceeds azure block limit
+            @current_index += 1
+            old_azure_storage_path = @azure_storage_path
+            generate_log_name(chunk, @current_index)
+
+            # If index is not a part of format, rethrow exception.
+            if old_azure_storage_path == @azure_storage_path
+              log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
+              raise
+            end
+
+            log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          elsif status_code == 404 # blob not found
+            log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          else
+            raise
+          end
+        end
+        log.debug 'azure_storage_append_blob: append_blob.complete'
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path('..', __dir__))
+require 'test-unit'
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -0,0 +1,197 @@
+require 'helper'
+require 'fluent/plugin/out_azure-storage-append-blob.rb'
+require 'azure/core/http/http_response'
+require 'azure/core/http/http_error'
+
+include Fluent::Test::Helpers
+
+class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  CONFIG = %(
+    azure_cloud AZUREGERMANCLOUD
+    azure_storage_account test_storage_account
+    azure_storage_access_key MY_FAKE_SECRET
+    azure_container test_container
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
+  CONNSTR_CONFIG = %(
+    azure_storage_connection_string https://test
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
+  MSI_CONFIG = %(
+    azure_storage_account test_storage_account
+    azure_container test_container
+    azure_imds_api_version 1970-01-01
+    azure_token_refresh_interval 120
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
+  def create_driver(conf: CONFIG, service: nil)
+    d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
+    d.instance.instance_variable_set(:@bs, service)
+    d.instance.instance_variable_set(:@azure_storage_path, 'storage_path')
+    d
+  end
+
+  sub_test_case 'test config' do
+    test 'config should reject with no azure container' do
+      assert_raise Fluent::ConfigError do
+        create_driver conf: %(
+          azure_storage_account test_storage_account
+          azure_storage_access_key MY_FAKE_SECRET
+          time_slice_format        %Y%m%d-%H
+          time_slice_wait          10m
+          path log
+        )
+      end
+    end
+
+    test 'config with access key should set instance variables' do
+      d = create_driver
+      assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+    end
+
+    test 'config with managed identity enabled should set instance variables' do
+      d = create_driver conf: MSI_CONFIG
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.instance_variable_get(:@use_msi)
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+      assert_equal 120, d.instance.azure_token_refresh_interval
+      assert_equal '1970-01-01', d.instance.azure_imds_api_version
+    end
+
+    test 'config with connection string should set instance variables' do
+      d = create_driver conf: CONNSTR_CONFIG
+      assert_equal 'https://test', d.instance.azure_storage_connection_string
+      assert_equal false, d.instance.instance_variable_get(:@use_msi)
+      assert_equal true, d.instance.auto_create_container
+    end
+  end
+
+  sub_test_case 'test path slicing' do
+    test 'test path_slicing' do
+      config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
+      d = create_driver conf: config
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
+    end
+
+    test 'path slicing utc' do
+      config = CONFIG.clone.gsub(/path\slog/, 'path log/%Y/%m/%d')
+      config << "\nutc\n"
+      d = create_driver conf: config
+      path_slicer = d.instance.instance_variable_get(:@path_slicer)
+      path = d.instance.instance_variable_get(:@path)
+      slice = path_slicer.call(path)
+      assert_equal slice, Time.now.utc.strftime('log/%Y/%m/%d')
+    end
+  end
+
+  # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
+  # a response object when it is created.
+  class FakeResponse
+    def initialize(status = 404)
+      @status = status
+      @body = 'body'
+      @headers = {}
+    end
+
+    attr_reader :status, :body, :headers
+  end
+
+  # This class is used to test plugin functions which interact with the blob service
+  class FakeBlobService
+    def initialize(status)
+      @response = Azure::Core::Http::HttpResponse.new(FakeResponse.new(status))
+      @blocks = []
+    end
+    attr_reader :blocks
+
+    def append_blob_block(_container, _path, data, options={})
+      @blocks.append(data)
+    end
+
+    def get_container_properties(_container)
+      unless @response.status_code == 200
+        raise Azure::Core::Http::HTTPError.new(@response)
+      end
+    end
+  end
+
+  sub_test_case 'test container_exists' do
+    test 'container 404 returns false' do
+      d = create_driver service: FakeBlobService.new(404)
+      assert_false d.instance.container_exists? 'anything'
+    end
+
+    test 'existing container returns true' do
+      d = create_driver service: FakeBlobService.new(200)
+      assert_true d.instance.container_exists? 'anything'
+    end
+
+    test 'unexpected exception raises' do
+      d = create_driver service: FakeBlobService.new(500)
+      assert_raise_kind_of Azure::Core::Http::HTTPError do
+        d.instance.container_exists? 'anything'
+      end
+    end
+  end
+
+  # Override the block size limit so that mocked requests do not require huge buffers
+  class Fluent::Plugin::AzureStorageAppendBlobOut
+    AZURE_BLOCK_SIZE_LIMIT = 10
+  end
+
+  sub_test_case 'test append blob buffering' do
+    def fake_appended_blocks(content)
+      # run the append on the fake blob service, return a list of append request buffers
+      svc = FakeBlobService.new(200)
+      d = create_driver service: svc
+      d.instance.send(:append_blob, content, nil)
+      svc.blocks
+    end
+
+    test 'short buffer appends once' do
+      content = '123456789'
+      blocks = fake_appended_blocks content
+      assert_equal [content], blocks
+    end
+
+    test 'single character appends once' do
+      content = '1'
+      blocks = fake_appended_blocks content
+      assert_equal [content], blocks
+    end
+
+    test 'empty appends once' do
+      content = ''
+      blocks = fake_appended_blocks content
+      assert_equal [''], blocks
+    end
+
+    test 'long buffer appends multiple times' do
+      limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
+      buf1 = 'a' * limit
+      buf2 = 'a' * 3
+      blocks = fake_appended_blocks buf1 + buf2
+      assert_equal [buf1, buf2], blocks
+    end
+  end
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azure-storage-append-blob-lts-azurestack
+version: !ruby/object:Gem::Version
+  version: 0.6.2
+platform: ruby
+authors:
+- Jiaxun Song
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: azure-storage-blob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
+  https://github.com/microsoft/fluent-plugin-azure-storage-append-blob
+email:
+- jiaxun.song@outlook.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/publish.yaml"
+- ".github/workflows/test.yaml"
+- ".gitignore"
+- ".rubocop.yml"
+- Dockerfile
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-azure-storage-append-blob-lts.gemspec
+- lib/fluent/plugin/out_azure-storage-append-blob.rb
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb
+homepage: https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.11
+signing_key:
+specification_version: 4
+summary: Azure Storage Append Blob output plugin for Fluentd event collector
+test_files:
+- test/helper.rb
+- test/plugin/test_out_azure_storage_append_blob.rb