fluent-plugin-azurestorage-gen2 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 857dcdb31e03f6564e547e0183a3bf24f06c8a0e
-  data.tar.gz: f1f1d048e69afba0f469a1999b6786473e282092
+  metadata.gz: 6f15931cc13f12c4aeb6723ba2a44c604998e819
+  data.tar.gz: 557964c5e84e0274a840af3e18500d9197ea8cd4
 SHA512:
-  metadata.gz: 6c87884fa508bc01f7470653b0a3b67d4a4cb02bddab03eb9071c1360f25abcf13558d1023a11be7d1277dcebf0794d4fb01ff19982a2000c2879ff335421773
-  data.tar.gz: 1491be525cbf66dbd7240ddea5a2ba33ded91e4c4c5816d238fc4ac49a76057e05b7bc7dea881220dc32fff836239d2d485175699285041bb1330c955b103e0b
+  metadata.gz: 7d0e97ee7d6b1429e98f91aa72135c4f84bf4cfd8b573cc8ec747cb36fe0e65fb98bce5d02d5416db83a4946c406c93f27ede48a1ba989d25bf675771311e6cd
+  data.tar.gz: 6ed3f231452a94eb0e66244648be66500af2fd82577b3796df11d0cda363238b95b50cc478ae01b54d1e8deb4cdb12101151ccfecb8177f53923db97b395248b

data/README.md

@@ -1,13 +1,15 @@
 # Azure Datalake Storage Gen2 Fluentd Output Plugin (IN PROGRESS)
 
 [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
-[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
+![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?extension=png)
 
 ## Requirements
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.1 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -22,11 +24,188 @@ $ gem install fluent-plugin-azurestorage-gen2
 
 ## Configuration
 
-TODO
+```
+<match **>
+  @type azurestorage_gen2
+  azure_storage_account            mystorageabfs
+  azure_container                  mycontainer
+  azure_instance_msi               /subscriptions/mysubscriptionid
+  azure_object_key_format          %{path}-%{index}.%{file_extension}
+  azure_oauth_refresh_interval     3600
+  time_slice_format                %Y%m%d-%H
+  file_extension                   log
+  path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
+  auto_create_container            true
+  <buffer tag,time>
+    @type file
+    path /var/log/fluent/azurestorage-buffer
+    timekey 5m
+    timekey_wait 0s
+    timekey_use_utc true
+    chunk_limit_size 64m
+  </buffer>
+</match>
+```
 
 ### Configuration options
 
-TODO
+### azure_storage_account
+
+Your Azure Storage Account Name. This can be got from Azure Management potal.
+This parameter is required when environment variable 'AZURE_STORAGE_ACCOUNT' is not set.
+
+### azure_storage_access_key (not implemented yet - use msi)
+
+Your Azure Storage Access Key(Primary or Secondary). This also can be got from Azure Management potal. Storage access key authentication is used when this parameter is provided or environment variable 'AZURE_STORAGE_ACCESS_KEY' is set.
+
+### azure_instance_msi
+
+Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
+
+### azure_oauth_refresh_interval
+
+OAuth2 access token refreshment interval in second. Only applies when MSI authentication is used.
+
+### azure_container (Required)
+
+Azure Storage Container name
+
+### auto_create_container
+
+This plugin create container if not exist when you set 'auto_create_container' to true.
+
+### azure_object_key_format
+
+The format of Azure Storage object keys. You can use several built-in variables:
+
+- %{path}
+- %{time_slice}
+- %{index}
+- %{file_extension}
+
+to decide keys dynamically.
+
+%{path} is exactly the value of *path* configured in the configuration file. E.g., "logs/" in the example configuration above.
+%{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
+%{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
+%{file_extention} is always "gz" for now.
+
+The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
+
+For instance, using the example configuration above, actual object keys on Azure Storage will be something like:
+
+```
+"logs/20130111-22_0.gz"
+"logs/20130111-23_0.gz"
+"logs/20130111-23_1.gz"
+"logs/20130112-00_0.gz"
+```
+
+With the configuration:
+
+```
+azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}.%{file_extension}
+path log
+time_slice_format %Y%m%d-%H
+```
+
+You get:
+
+```
+"log/events/ts=20130111-22/events_0.gz"
+"log/events/ts=20130111-23/events_0.gz"
+"log/events/ts=20130111-23/events_1.gz"
+"log/events/ts=20130112-00/events_0.gz"
+```
+
+The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders) mixin is also incorporated, so additional variables such as %{hostname}, %{uuid}, etc. can be used in the azure_object_key_format. This could prove useful in preventing filename conflicts when writing from multiple servers.
+
+```
+azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostname}.%{file_extension}
+```
+
+### format
+
+Change one line format in the Azure Storage object. Supported formats are 'out_file', 'json', 'ltsv' and 'single_value'.
+
+- out_file (default)
+
+```
+time\ttag\t{..json1..}
+time\ttag\t{..json2..}
+...
+```
+
+- json
+
+```
+{..json1..}
+{..json2..}
+...
+```
+
+At this format, "time" and "tag" are omitted.
+But you can set these information to the record by setting "include_tag_key" / "tag_key" and "include_time_key" / "time_key" option.
+If you set following configuration in AzureStorage output:
+
+```
+format json
+include_time_key true
+time_key log_time # default is time
+```
+
+then the record has log_time field.
+
+```
+{"log_time":"time string",...}
+```
+
+- ltsv
+
+```
+key1:value1\tkey2:value2
+key1:value1\tkey2:value2
+...
+```
+
+"ltsv" format also accepts "include_xxx" related options. See "json" section.
+
+- single_value
+
+Use specified value instead of entire recode. If you get '{"message":"my log"}', then contents are
+
+```
+my log1
+my log2
+...
+```
+
+You can change key name by "message_key" option.
+
+### path
+
+path prefix of the files on Azure Storage. Default is "" (no prefix).
+
+### buffer_path (required)
+
+path prefix of the files to buffer logs.
+
+### time_slice_format
+
+Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
+
+### time_slice_wait
+
+The time to wait old logs. Default is 10 minutes.
+
+### utc
+
+Use UTC instead of local time.
+
+## TODOs
+
+- add storage key support
+- add compression (if append is not used)
 
 ## Contributing
 

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/fluent-plugin-azurestorage-gen2.gemspec

@@ -17,8 +17,6 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
-  gem.add_runtime_dependency 'azure-storage-common', '~> 1.1', '>= 1.1.0'
-  gem.add_runtime_dependency 'azure-storage-blob', '~> 1.1', '>= 1.1.0'
   gem.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
   gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
   gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -235,12 +235,12 @@ module Fluent::Plugin
         def create_blob(blob_path)
             headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"Authorization" => "Bearer #{@azure_access_token}",:"Content-Length" => "0", :"Content-Type" => "application/json"}
             params = {:resource => "file", :recursive => "false"}
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}/testprefix#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating blob '#{@blob_path}' request timed out."
+                    raise Fluent::UnrecoverableError,  "Creating blob '#{blob_path}' request timed out."
                 elsif response.code == 409
                     log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
                 else
@@ -260,7 +260,7 @@ module Fluent::Plugin
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Appending blob #{@blob_path}' request timed out."
+                    raise Fluent::UnrecoverableError,  "Appending blob #{blob_path}' request timed out."
                 elsif response.code == 404
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
                 elsif response.code == 409
@@ -282,7 +282,7 @@ module Fluent::Plugin
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Bloub '#{@blob_path}' flush request timed out."
+                    raise Fluent::UnrecoverableError,  "Bloub '#{blob_path}' flush request timed out."
                 else
                     raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}"
                 end
@@ -301,12 +301,12 @@ module Fluent::Plugin
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
                   content_length = response.headers['Content-Length'].to_i
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Get blob properties '#{@blob_path}' request timed out."
+                    raise Fluent::UnrecoverableError,  "Get blob properties '#{blob_path}' request timed out."
                 elsif response.code == 404
-                    log.debug "azurestorage_gen2: Blob '#{@blob_path}' does not exist. Creating it if needed..."
+                    log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
                     content_length = 0
                 else
-                    raise Fluent::UnrecoverableError, "Get blob properties '#{@blob_path}' request failed - code: #{response.code}, body: #{response.body}"
+                    raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}"
                 end
             end
             request.run

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-15 00:00:00.000000000 Z
+date: 2019-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,46 +30,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: azure-storage-common
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.0
-- !ruby/object:Gem::Dependency
-  name: azure-storage-blob
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: uuidtools
   requirement: !ruby/object:Gem::Requirement