logstash-output-adls 1.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e2df6f39f6459b1e43ffdb634da8179e0f1437a
+  data.tar.gz: 0c02a2b820c60277efba33a3bf7379d46fca5c87
+SHA512:
+  metadata.gz: ba9598c0b5a37907bc817a986724a11e93d0316763d3c62804c1048b01929459773b8adbd6dede821811604c2c684822232b98de8b26bf9c0a545848b25a529e
+  data.tar.gz: 79bb890474237a38d94c6c0957d14b56cb9f7ba751c99b20a4f8020bae7262d789e3f550f7c6dc3f990d552f6fe72a2bf5c6022608d7aba9f5d83dc85847c2f3

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2017 NOS Inovacao <http://www.nos.pt>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,4 @@
+Copyright (c) 2017 NOS Inovacao <http://www.nos.p
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,119 @@
+# Azure Data Lake Store Output Logstash Plugin
+
+This is a Azure Data Laka Store Output Plugin for [Logstash](https://github.com/elastic/logstash).
+
+This plugin uses the official [Microsoft Data Lake Store Java SDK](https://github.com/Azure/azure-data-lake-store-java) with their custom [AzureDataLakeFilesystem - ADL](https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-overview#what-is-azure-data-lake-store-file-system-adl) protocol, which Microsoft claims is more efficient than WebHDFS.
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Installing
+
+TBD
+
+### Manual installation on a already deployed Logstash:
+
+You can install the plugin on a already deployed Logstash and avoid messing with the Logstash Gemfile. 
+Please note that in this mode you can't alter the plugin source (logstash-output-adls/lib/logstash/outputs/adls.rb) without rebuilding the gem, which makes sense for a production deploy.
+
+Note that you'll need an already built gem file from the previous step.
+
+- Copy logstash-output-adls-x.x.x.gem to your remote Logstash.
+- Install the package on your remote Logstash:
+
+```sh
+bin/logstash-plugin install /your/local/logstash-output-adls-x.x.x.gem
+```
+
+## Configuration
+
+### Configuration example:
+
+ ```
+ input {
+   ...
+ }
+ filter {
+   ...
+ }
+ output {
+   adls {
+     adls_fqdn => "XXXXXXXXXXX.azuredatalakestore.net"                                        # (required)
+     adls_token_endpoint => "https://login.microsoftonline.com/XXXXXXXXXX/oauth2/token"       # (required)
+     adls_client_id => "00000000-0000-0000-0000-000000000000"                                 # (required)
+     adls_client_key => "XXXXXXXXXXXXXXXXXXXXXX"                                              # (required)
+     path => "/logstash/%{+YYYY}/%{+MM}/%{+dd}/logstash-%{+HH}-%{[@metadata][cid]}.log"       # (required)
+     line_separator => "\n"                                                                   # (optional, default: "\n")
+     created_files_permission => 755                                                          # (optional, default: 755)
+     adls_token_expire_security_margin => 300                                                 # (optional, default: 300)
+     single_file_per_thread => true                                                           # (optional, default: true)
+     retry_interval => 0.5                                                                    # (optional, default: 0.5)
+     max_retry_interval => 10                                                                 # (optional, default: 10)
+     retry_times => 3                                                                         # (optional, default: 3)
+     exit_if_retries_exceeded => false                                                        # (optional, default: false)
+     codec => "json"                                                                          # (optional, default: default codec defined by Logstash)
+   }
+ }
+```
+
+### Configuration fields:
+
+| Setting | Required | Default | Description |
+| --- | --- | --- | --- |
+| `adls_fqdn` | yes | | Azure DLS FQDN |
+| `adls_token_endpoint` | yes | | Azure Oauth Endpoint |
+| `adls_client_id` | yes | | Azure DLS ClientID |
+| `adls_client_key` | yes | | Azure DLS ClientKey |
+| `path` | yes | | The path to the file to write to. Event fields can be used here, as well as date fields in the joda time format, e.g.: `/logstash/%{+YYYY-MM-dd}/logstash-%{+HH}-%{[@metadata][cid]}.log` |
+| `line_separator` | no | `\n` | Line separator for events written |
+| `created_files_permission` | no | 755 | File permission for files created |
+| `adls_token_expire_security_margin` | no | 300 | The security margin (in seconds) that shoud be subtracted to the token's expire value to calculate when the token shoud be renewed. (i.e. If the Oauth token expires in 1hour, it will be renewed in "1hour -adls_token_expire_security_margin " |
+| `single_file_per_thread` | no | true | Avoid appending to same file in multiple threads. This solves some problems with multiple logstash output threads and locked file leases in ADLS. If this option is set to true, %{[@metadata][cid]} needs to be used in path config settting. %{[@metadata][cid]} (cid->concurrentId) is generated from a random value computed when the logstash instance starts plus a per thread id. This setting is used to deal with ADLS 0x83090a16 errors. (see Configuration notes) |
+| `retry_interval` | no | 1 | How long(in seconds)  should we wait between retries in case of an error. This value is a coefficient and not an absolute value. The wait time is "retry_interval*tries_counter". So, if retry_interval is 1 on the first retry, the wait time will be 1, on the second try will be 2, and so on |
+| `max_retry_interval` | no | 10 | Max Retry Interval. The actual wait time (in seconds) will be min(retry_interval*tries_counter", max_retry_interval) |
+| `retry_times` | no | 3 | How many times should we retry. If retry_times is exceeded, an error will be logged and the event will be discarded. (Set to -1 for unlimited retries) |
+| `exit_if_retries_exceeded` | no | false | If enabled, Logstash will exit if retries are exceeded to avoid loosing events |
+| `codec` | no | Logstash default codec | The Codec that will be used to serialize the event.(ex: CSV, JSON, LINE, etc) If you do not define one, Logstash will use it's default. Please refer to Logstash documentation |
+
+
+### Concurrency and batching:
+This plugin relies only on Logstash concurrency/batching facilities and can configured by Logstash's own "pipeline.workers" and "pipeline.batch.size" settings. Also, the concurrency mode of this plugin is set to [shared](https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html#_see_what_your_plugin_looks_like_4) to maximize concurrency.
+
+### Configuration notes:
+
+- If **single_file_per_thread** is enabled (and it is by defaut) and you're using more than one working thread, you'll need to add **%{[@metadata][cid]}** to your file path. This concatenates a ConcurrentID value to your path to avoid remote concurrency problems in ADLS. Apparently, ADLS locks the file for writing.
+- If you you still have errors in your log like "**APPEND failed with error 0x83090a16 (Internal server error.)**" you should lower your concurrency and/or batching settings to avoid these kind of errors. We try to mitigate the problem with a backoff and retry strategy (**retry_interval** and **retry_times** settings) but AFAIK, there's nothing this plugin can do to avoid that entirely, it's an ADLS problem. Maybe they should queue their writing requests internally instead of writing them directly to the FS to avoid file write locks???
+- However, unless you have very high concurrency and/or a large batch size, these errors shouldn't be a problem.
+
+
+## Build & Development
+
+### Build
+- To get started, you'll need JRuby with the Bundler and Rake gems installed.
+
+- Install dependencies:
+```sh
+bundle install
+```
+
+- Install Java dependencies from Maven:
+```sh
+rake install_jars
+```
+
+- Build your plugin gem:
+
+```sh
+gem build logstash-output-adls.gemspec
+```
+
+### Run in a local Logstash for development purposes.
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```sh
+gem "logstash-output-adls", :path => "/your/local/logstash-output-adls"
+```
+
+- Install plugin:
+```sh
+bin/logstash-plugin install --no-verify
+```

data/lib/logstash-output-adls_jars.rb

@@ -0,0 +1,18 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar'
+  require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
+  require 'org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar'
+  require 'log4j/log4j/1.2.17/log4j-1.2.17.jar'
+  require 'com/microsoft/azure/azure-data-lake-store-sdk/2.1.1/azure-data-lake-store-sdk-2.1.1.jar'
+end
+
+if defined? Jars
+  require_jar( 'com.fasterxml.jackson.core', 'jackson-core', '2.7.4' )
+  require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
+  require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.21' )
+  require_jar( 'log4j', 'log4j', '1.2.17' )
+  require_jar( 'com.microsoft.azure', 'azure-data-lake-store-sdk', '2.1.1' )
+end

data/lib/logstash/outputs/adls.rb

@@ -0,0 +1,239 @@
+# encoding: utf-8
+require 'date'
+require "logstash/namespace"
+require 'logstash/outputs/base'
+require 'java'
+require 'logstash-output-adls_jars.rb'
+
+# ==== Usage
+# This is an example of Logstash config:
+#
+# [source,ruby]
+# ----------------------------------
+# input {
+#   ...
+# }
+# filter {
+#   ...
+# }
+# output {
+#   adls {
+#     adls_fqdn => "XXXXXXXXXXX.azuredatalakestore.net"                                         # (required)
+#     adls_token_endpoint => "https://login.microsoftonline.com/XXXXXXXXXX/oauth2/token"        # (required)
+#     adls_client_id => "00000000-0000-0000-0000-000000000000"                                  # (required)
+#     adls_client_key => "XXXXXXXXXXXXXXXXXXXXXX"                                               # (required)
+#     path => "/logstash/%{+YYYY}/%{+MM}/%{+dd}/logstash-%{+HH}-%{[@metadata][cid]}.log"        # (required)
+#     line_separator => "\n"                                                                    # (optional, default: "\n")
+#     created_files_permission => 755                                                           # (optional, default: 755)
+#     adls_token_expire_security_margin => 300                                                  # (optional, default: 300)
+#     single_file_per_thread = > true                                                           # (optional, default: true)
+#     retry_interval => 0.5                                                                     # (optional, default: 0.5)
+#     max_retry_interval => 10                                                                  # (optional, default: 10)
+#     retry_times => 3                                                                          # (optional, default: 3)
+#     exit_if_retries_exceeded => false                                                         # (optional, default: false)
+#     codec => "json"                                                                           # (optional, default: default codec defined by Logstash)
+#   }
+# }
+# ----------------------------------
+class LogStash::Outputs::ADLS < LogStash::Outputs::Base
+
+  config_name "adls"
+
+  concurrency :shared
+
+  # The Azure DLS FQDN
+  config :adls_fqdn, :validate => :string, :required => true
+
+  # The Azure Oauth Endpoint
+  config :adls_token_endpoint, :validate => :string, :required => true
+
+  # The Azure DLS ClientID
+  config :adls_client_id, :validate => :string, :required => true
+
+  # The  The Azure DLS ClientKey
+  config :adls_client_key, :validate => :string, :required => true
+
+  # The path to the file to write to. Event fields can be used here,
+  # as well as date fields in the joda time format, e.g.:
+  # `/user/logstash/dt=%{+YYYY-MM-dd}/%{@source_host}-%{+HH}.log`
+  config :path, :validate => :string, :required => true
+
+  # Line separator for events written.
+  config :line_separator, :validate => :string, :default => "\n"
+
+  # File permission for files created
+  config :created_files_permission, :validate => :number, :default => 755
+
+  # The security margin that shoud be subtracted to the token's expire value.
+  config :adls_token_expire_security_margin, :validate => :number, :default => 300
+  
+  # Avoid appending to same file in multiple threads.
+  # This solves some problems with multiple logstash output threads and locked file leases in webhdfs.
+  # If this option is set to true, %{[@metadata][cid]} needs to be used in path config settting.
+  config :single_file_per_thread, :validate => :boolean, :default => true
+
+  # How long(in seconds) should we wait between retries in case of an error. This value is a coefficient 
+  # and not an absolute value. The wait time is "retry_interval*tries_counter". So, if retry_interval is 1 
+  # on the first retry, the wait time will be 1, on the second try will be 2, and so on.
+  config :retry_interval, :validate => :number, :default => 1
+
+  # Max Retry Interval. The actual wait time (in seconds) will be min(retry_interval*tries_counter", max_retry_interval).
+  config :max_retry_interval, :validate => :number, :default => 10
+
+  # How many times should we retry. If retry_times is exceeded, an error will be logged and the event will be discarded. (Set to -1 for unlimited retries)
+  config :retry_times, :validate => :number, :default => 3
+
+  # If enabled, Logstash will exit if retries are exceeded to avoid loosing events.
+  config :exit_if_retries_exceeded, :validate => :boolean, :default => false
+
+  attr_accessor :client
+  attr_accessor :azureOauthTokenRefreshDate
+  attr_accessor :timerTaskClass
+  attr_accessor :timer
+  
+  public
+
+    def register()
+
+      begin
+        @client = prepare_client(@adls_fqdn, @adls_client_id, @adls_token_endpoint, @adls_client_key)
+      rescue => e
+        logger.error("Cannot Login in ADLS. Aborting.... Exception: #{e.message}; Trace:#{e.backtrace.join("\n\t")}")
+        exit 1
+      end
+
+      # Make sure @path contains %{[@metadata][thread_id]} format value
+      if @single_file_per_thread and !@path.include? "%{[@metadata][cid]}"
+        @logger.error("Please set %{[@metadata][cid]} format value in @path to avoid file locks in ADLS.")
+        raise LogStash::ConfigurationError
+      end
+
+      @codec.on_event do |event, encoded_event|
+        encoded_event
+      end
+
+      @timerTaskClass = Class.new java.util.TimerTask do
+        def setContext(parent)
+          @parent = parent
+        end
+        def run
+          begin
+            @parent.client = @parent.prepare_client(@parent.adls_fqdn, @parent.adls_client_id, @parent.adls_token_endpoint, @parent.adls_client_key)
+          rescue => e
+            sleepTime = [@parent.retry_interval, @parent.max_retry_interval].min
+            @parent.logger.error("ADLS Refresh OAuth Token Failed! Retrying in #{sleepTime.to_s} seconds... Exception: #{e.message}; Trace:#{e.backtrace.join("\n\t")}")         
+            sleep(sleepTime)
+          end
+          timerTask = @parent.timerTaskClass.new
+          timerTask.setContext(@parent)
+          @parent.timer.schedule(timerTask, @parent.azureOauthTokenRefreshDate) # Rearm timer
+        end  
+      end  
+
+      timerTask = @timerTaskClass.new
+      timerTask.setContext(self)
+
+      @timer = java.util.Timer.new
+      @timer.schedule(timerTask, @azureOauthTokenRefreshDate)
+
+      @randomValuePerInstance = rand(10..10000) # To make sure different instances in different machines don't generate the same threadId.
+  end
+
+  def close
+    @logger.info("Logstash ADLS output plugin is shutting down...")
+  end
+
+  def prepare_client(accountFQDN, clientId, authTokenEndpoint, clientKey)
+    azureToken = com.microsoft.azure.datalake.store.oauth2.AzureADAuthenticator.getTokenUsingClientCreds(authTokenEndpoint, clientId, clientKey)
+
+    calendar = java.util.Calendar.getInstance()
+    calendar.setTime(azureToken.expiry)
+    calendar.set(java.util.Calendar::SECOND,(calendar.get(java.util.Calendar::SECOND)-@adls_token_expire_security_margin))
+    @azureOauthTokenRefreshDate = calendar.getTime()
+
+    @logger.info("Got ADLS OAuth Token with expire date #{azureToken.expiry.to_s}. Token will be refreshed at #{@azureOauthTokenRefreshDate.to_s}")
+    
+    client = com.microsoft.azure.datalake.store.ADLStoreClient.createClient(accountFQDN, azureToken)
+    options = com.microsoft.azure.datalake.store.ADLStoreOptions.new()
+    options.setUserAgentSuffix("Logstash-ADLS-Output-Plugin")
+    client.setOptions(options)
+    client.checkExists("testFile") # Test the Client to make sure it works. The return value is irrelevant. 
+    client
+  end
+
+  def multi_receive(events)
+    return if not events
+
+    timeElapsed = Time.now
+
+    output_files = Hash.new { |hash, key| hash[key] = "" }
+    events.collect do |event|
+
+      if @single_file_per_thread
+        event.set("[@metadata][cid]", "#{@randomValuePerInstance.to_s}#{Thread.current.object_id.to_s}")
+      end
+
+      path = event.sprintf(@path)
+      event_as_string = @codec.encode(event)
+      event_as_string +=  @line_separator unless event_as_string.end_with?  @line_separator
+      output_files[path] << event_as_string
+    end
+  
+    output_files.each do |path, output|
+      # Retry max_retry times. This can solve problems like leases being hold by another process.
+      write_tries = 0
+      begin
+        write_data(path, output)
+      rescue Exception => e
+        if (write_tries < @retry_times) or (@retry_times == -1)
+          sleepTime = [@retry_interval * write_tries, @max_retry_interval].min
+          @logger.warn("ADLS write caused an exception: #{e.message}. Maybe you should increase retry_interval or reduce number of workers. Attempt: #{write_tries.to_s}. Retrying in #{sleepTime.to_s} seconds...")
+          sleep(sleepTime)
+          write_tries += 1
+          retry
+        else
+          if e.instance_of? com.microsoft.azure.datalake.store.ADLException
+            @logger.error("Max write retries reached. Events discarded! ADLS_RemoteMessage: #{e.remoteExceptionMessage}; Exception: #{e.message};  ADLS_Path: #{path}; StackTrace:#{e.backtrace.join("\n\t")}")
+          else
+            @logger.error("Max write retries reached. Events discarded! Exception: #{e.message}; StackTrace:#{e.backtrace.join("\n\t")}")
+          end
+          if @exit_if_retries_exceeded
+            exit 1
+          end
+        end
+      end 
+    end
+    @logger.debug("#{events.length.to_s} events written on ADLS in #{Time.now-timeElapsed} seconds.")
+
+  end
+
+  def write_data(path, data)
+    begin
+      #@logger.info("Trying to write at #{path}")
+      adlsClient = @client
+      
+      # Try to append to already existing file, which will work most of the times.
+      stream = adlsClient.getAppendStream(path)
+      outStream = java.io.PrintStream.new(stream)
+      outStream.print(data)
+      outStream.close()
+      stream.close()
+
+    # File does not exist, so create it.
+    rescue com.microsoft.azure.datalake.store.ADLException => e
+      if e.httpResponseCode == 404
+        createStream = adlsClient.createFile(path, com.microsoft.azure.datalake.store.IfExists::OVERWRITE, @created_files_permission.to_s, true)
+        outStream = java.io.PrintStream.new(createStream)
+        outStream.print(data)
+        outStream.close()
+        createStream.close()
+        @logger.debug("File #{path} created.")
+      else
+         raise e  
+      end           
+    end
+    #@logger.info("Data written to ADLS: #{data}")
+
+  end
+        
+end

data/logstash-output-adls.gemspec

@@ -0,0 +1,37 @@
+# encoding: utf-8
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-output-adls'
+  s.version         = '1.1.3'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "Plugin to write events to Azure DataLakeStore"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["NOS Inovacao"]
+  s.email           = 'nosi.metadata@nos.pt'
+  s.homepage        = "http://www.nos.pt"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md', 'Gemfile','LICENSE','NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  s.requirements << "jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.1'"
+  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
+
+  s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_development_dependency 'logstash-devutils'
+
+  s.add_development_dependency 'logstash-codec-line'
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'logstash-codec-plain'
+
+
+end

data/spec/integration/adls_spec.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/adls'
+require 'json'
+require 'java'
+
+describe LogStash::Outputs::ADLS, :integration => true do
+
+  let(:adls_fqdn) { 'XXXXXXXXXXX.azuredatalakestore.net' }
+  let(:adls_token_endpoint) { 'https://login.microsoftonline.com/XXXXXXXXXX/oauth2/token' }
+  let(:adls_client_id) { '00000000-0000-0000-0000-000000000000' }
+  let(:adls_client_key) { 'XXXXXXXXXXXXXXXXXXXXXX' }
+  let(:path) { '/test.log' }
+
+  let(:config) { { 'adls_fqdn' =>adls_fqdn, 'adls_token_endpoint' => adls_token_endpoint, 'adls_client_id' => adls_client_id, 'adls_client_key' => adls_client_key, 'path' => path, "single_file_per_thread" => false } }
+
+  subject(:plugin) { LogStash::Plugin.lookup("output", "adls").new(config) }
+
+  let(:event) { LogStash::Event.new('message' => 'Hello world!', 'source' => 'out of the blue',
+                                    'type' => 'generator', 'host' => 'localhost' ) }
+
+  describe "register and close" do
+
+    it 'should register with default values' do
+      expect { subject.register }.to_not raise_error
+    end
+
+  end
+
+  describe '#write' do
+
+    let(:AdlsClient) { nil }
+
+    after(:each) do
+      subject.close
+      #deltefile
+    end
+
+    describe "writing plain files" do
+
+      before(:each) do
+        subject.register
+        AdlsClient = subject.client
+        subject.receive(event)
+      end
+
+      it 'should use the correct filename pattern' do
+        expect { AdlsClient.checkExists(path) }.to eq(true)
+      end
+
+      context "using the line codec without format" do
+
+        it 'should match the event data' do
+
+          expected = expect do
+            stream = AdlsClient.getReadStream(path)
+            s = java.util.Scanner(stream).new.useDelimiter("\\A")
+            result = s.hasNext() ? s.next() : ""
+            result
+          end
+          expected.to eq(event.to_s)
+        end
+
+      end
+
+      #context "using the json codec" do
+
+        #let(:config) { { 'host' => host, 'user' => user, 'flush_size' => 10, 'path' => test_file, 'compression' => 'none', 'codec' => 'json' } }
+
+        #it 'should match the event data' do
+          #expect(webhdfs_client.read(hdfs_file_name).strip()).to eq(event.to_json)
+        #end
+
+      #end
+
+    end
+
+  end
+end

data/spec/outputs/adls_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/adls'
+
+describe 'outputs/adls' do
+
+  let(:adls_fqdn) { 'XXXXXXXXXXX.azuredatalakestore.net' }
+  let(:adls_token_endpoint) { 'https://login.microsoftonline.com/XXXXXXXXXX/oauth2/token' }
+  let(:adls_client_id) { '00000000-0000-0000-0000-000000000000' }
+  let(:adls_client_key) { 'XXXXXXXXXXXXXXXXXXXXXX' }
+  let(:path) { '/test.log' }
+
+  let(:config) { { 'adls_fqdn' =>adls_fqdn, 'adls_token_endpoint' => adls_token_endpoint, 'adls_client_id' => adls_client_id, 'adls_client_key' => adls_client_key, 'path' => path } }
+
+  subject(:plugin) { LogStash::Plugin.lookup("output", "adls").new(config) }
+
+  describe '#initializing' do
+
+    it 'should fail to register without %{[@metadata][cid]} in the path' do
+      plugin = LogStash::Plugin.lookup("output", "adls")
+      expect { plugin.new }.to raise_error(error=LogStash::ConfigurationError)
+    end
+
+    context "default values" do
+
+      it 'should have default line_separator' do
+        expect(subject.line_separator).to eq("\n")
+      end
+
+      it 'should have default created_files_permission' do
+        expect(subject.created_files_permission).to eq(755)
+      end
+      it 'should have default adls_token_expire_security_margin' do
+        expect(subject.adls_token_expire_security_margin).to eq(300)
+      end
+
+      it 'should have default single_file_per_thread' do
+        expect(subject.single_file_per_thread).to eq(true)
+      end
+
+      it 'should have default retry_interval' do
+        expect(subject.retry_interval).to eq(1)
+      end
+
+      it 'should have default max_retry_interval' do
+        expect(subject.max_retry_interval).to eq(10)
+      end
+
+      it 'should have default retry_times' do
+        expect(subject.retry_times).to eq(3)
+      end
+
+      it 'should have default exit_if_retries_exceeded' do
+        expect(subject.exit_if_retries_exceeded).to eq(false)
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-adls
+version: !ruby/object:Gem::Version
+  version: 1.1.3
+platform: ruby
+authors:
+- NOS Inovacao
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-03-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+email: nosi.metadata@nos.pt
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar
+- lib/com/microsoft/azure/azure-data-lake-store-sdk/2.1.1/azure-data-lake-store-sdk-2.1.1.jar
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
+- lib/logstash-output-adls_jars.rb
+- lib/logstash/outputs/adls.rb
+- lib/org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
+- logstash-output-adls.gemspec
+- spec/integration/adls_spec.rb
+- spec/outputs/adls_spec.rb
+- vendor/jar-dependencies/runtime-jars/azure-data-lake-store-sdk-2.1.1.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.7.4.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.21.jar
+homepage: http://www.nos.pt
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.1'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.21'
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Plugin to write events to Azure DataLakeStore
+test_files:
+- spec/integration/adls_spec.rb
+- spec/outputs/adls_spec.rb