logstash-output-adls 1.1.4 → 1.2.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58a94392f5ec4079d7df90c7da77bc893ebfe074
-  data.tar.gz: 35bbdf831f915708527e3d8c5f79bf510d293ef0
+  metadata.gz: 4d0ca693187aebbcce25fd9204e52838508f49fc
+  data.tar.gz: 67cbee093c81eaedf80cc071b162844434e56c59
 SHA512:
-  metadata.gz: 425471f3b7e34f1deca8682c7d309d9e4ed1a069349204cd21d60ec347c2616255e242f32808cae23f3dae4c888a74543cc1a0193006096e10c122bedbadc982
-  data.tar.gz: b78e4e0e8608b6ec7ad51b694a43c5da415b59c0472c23c562c4760d0727c00544c929ab3476d1569b03eb100adfb6fc4435ccb71c23c2ae6a361e5414513cc9
+  metadata.gz: a317642002d8a319ff45313bb7447b27ca718e6c6b94fb16438883234e4b716ea0f7f691fcff80a4a182c4ce2e912b00cc5157994ef744890a8d449fdbbedcec
+  data.tar.gz: 3725c0d76c6ebc9bd6f468d4ea55a6470710e47a5cd94a9f4ca33987e07f27536a32bfcd7a973fc5a227032da0a4ae018c3dd36147272d529190aa9e76422f4d

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 1.2.0
+ - Upgraded azure-data-lake-store-sdk to version 2.1.5 (from version 2.1.1).
+ - Build process moved to TravisCI
+ - Added debug logging when writing data to ADL
+
+# 1.1.4
+* First version of the adls plugin output

data/CONTRIBUTORS

@@ -0,0 +1,14 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Maintainers:
+* Carlos Alexandre
+
+Contributors:
+* Carlos Alexandre (calexandre)
+* João Serra (jserra)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -1,2 +1,11 @@
 source 'https://rubygems.org'
-gemspec
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/NOTICE.TXT

@@ -1,4 +1,4 @@
-Copyright (c) 2017 NOS Inovacao <http://www.nos.p
+Copyright (c) 2017 NOS Inovacao <http://www.nosinosvacao.pt>
 
 This product includes software developed by The Apache Software
 Foundation (http://www.apache.org/).

data/README.md

@@ -1,32 +1,15 @@
 # Azure Data Lake Store Output Logstash Plugin
 
+
+[![Travis Build Status](https://travis-ci.org/nosinovacao/logstash-output-adls.svg)](https://travis-ci.org/nosinovacao/logstash-output-adls)
+
 This is a Azure Data Laka Store Output Plugin for [Logstash](https://github.com/elastic/logstash).
 
 This plugin uses the official [Microsoft Data Lake Store Java SDK](https://github.com/Azure/azure-data-lake-store-java) with their custom [AzureDataLakeFilesystem - ADL](https://docs.microsoft.com/en-us/azure/data-lake-store/data-lake-store-overview#what-is-azure-data-lake-store-file-system-adl) protocol, which Microsoft claims is more efficient than WebHDFS.
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
-## Installing
-
-```sh
-bin/logstash-plugin install logstash-output-adls
-```
-
-### Manual installation on a already deployed Logstash:
-
-You can install the plugin on a already deployed Logstash and avoid messing with the Logstash Gemfile. 
-Please note that in this mode you can't alter the plugin source (logstash-output-adls/lib/logstash/outputs/adls.rb) without rebuilding the gem, which makes sense for a production deploy.
-
-Note that you'll need an already built gem file from the previous step.
-
-- Copy logstash-output-adls-x.x.x.gem to your remote Logstash.
-- Install the package on your remote Logstash:
-
-```sh
-bin/logstash-plugin install /your/local/logstash-output-adls-x.x.x.gem
-```
-
-## Configuration
+## Documentation
 
 ### Configuration example:
 
@@ -44,6 +27,7 @@ bin/logstash-plugin install /your/local/logstash-output-adls-x.x.x.gem
      adls_client_id => "00000000-0000-0000-0000-000000000000"                                 # (required)
      adls_client_key => "XXXXXXXXXXXXXXXXXXXXXX"                                              # (required)
      path => "/logstash/%{+YYYY}/%{+MM}/%{+dd}/logstash-%{+HH}-%{[@metadata][cid]}.log"       # (required)
+     test_path => "testfile"                                                                  # (optional, default "testfile")
      line_separator => "\n"                                                                   # (optional, default: "\n")
      created_files_permission => 755                                                          # (optional, default: 755)
      adls_token_expire_security_margin => 300                                                 # (optional, default: 300)
@@ -66,6 +50,7 @@ bin/logstash-plugin install /your/local/logstash-output-adls-x.x.x.gem
 | `adls_client_id` | yes | | Azure DLS ClientID |
 | `adls_client_key` | yes | | Azure DLS ClientKey |
 | `path` | yes | | The path to the file to write to. Event fields can be used here, as well as date fields in the joda time format, e.g.: `/logstash/%{+YYYY-MM-dd}/logstash-%{+HH}-%{[@metadata][cid]}.log` |
+| `test_path` | no | `testfile` | Path to test access on ADLS. This is used upon startup to ensure you have write access. |
 | `line_separator` | no | `\n` | Line separator for events written |
 | `created_files_permission` | no | 755 | File permission for files created |
 | `adls_token_expire_security_margin` | no | 300 | The security margin (in seconds) that shoud be subtracted to the token's expire value to calculate when the token shoud be renewed. (i.e. If the Oauth token expires in 1hour, it will be renewed in "1hour -adls_token_expire_security_margin " |
@@ -87,9 +72,48 @@ This plugin relies only on Logstash concurrency/batching facilities and can conf
 - However, unless you have very high concurrency and/or a large batch size, these errors shouldn't be a problem.
 
 
-## Build & Development
+## Developing
+
+### 1. Plugin Development and Testing
+
+To install, build and test, just run this command from the project's folder. If you want to deep-dive read into the next sections.
+
+```sh
+ci/build.sh
+```
+
+#### Build using Docker
+
+- If you want to use Docker to build, you can use Windows or Linux variants.
+- Execute the following commands in your project folder
+- The main advantage of using the method below, is that the container will preserve its state until it is removed, making subsequent `bundle install` commands much faster.
+
+**Windows (powershell):**
+```powershell
+# Start the container (it will stop once you exit it)
+"docker run -it -v $($pwd -replace "\\", "/")/:/project/ -w /project/ --name jruby jruby:latest bash" | Invoke-Expression
+
+# Start it again (it should stay up now)
+docker start jruby
+
+# Enter the container shell and execute the usual build commands
+docker exec -it jruby bash
+```
+
+**Linux**
+```sh
+# Start the container (it will stop once you exit it)
+docker run -it -v $pwd:/project -w /project --name jruby jruby:latest bash
+
+# Start it again (it should stay up now)
+docker start jruby
+
+# Enter the container shell and execute the usual build commands
+docker exec -it jruby bash
+```
+
 
-### Build
+#### Build
 - To get started, you'll need JRuby with the Bundler and Rake gems installed.
 
 - Install dependencies:
@@ -99,7 +123,7 @@ bundle install
 
 - Install Java dependencies from Maven:
 ```sh
-rake install_jars
+rake vendor
 ```
 
 - Build your plugin gem:
@@ -108,7 +132,28 @@ rake install_jars
 gem build logstash-output-adls.gemspec
 ```
 
-### Run in a local Logstash for development purposes.
+#### Test
+
+- Update your dependencies
+```
+bundle install
+```
+
+- Run unit tests
+```
+bundle exec rspec
+```
+
+- Run integration tests
+you'll need to have docker available within your test environment before running the integration tests. The tests depend on a specific Kafka image found in Docker Hub called spotify/kafka. You will need internet connectivity to pull in this image if it does not already exist locally.
+
+```
+bundle exec rspec --tag integration
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### Running your unpublished Plugin in Logstash
 
 - Edit Logstash `Gemfile` and add the local plugin path, for example:
 ```sh
@@ -117,5 +162,30 @@ gem "logstash-output-adls", :path => "/your/local/logstash-output-adls"
 
 - Install plugin:
 ```sh
+# Logstash 2.3 and higher
 bin/logstash-plugin install --no-verify
-```
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+```
+
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'input { stdin { } } output { adls { } }'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-output-adls.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+# Logstash 2.3 and higher
+bin/plugin install /your/local/plugin/logstash-output-adls.gem
+```
+- Start Logstash and proceed to test the plugin

data/lib/logstash-output-adls_jars.rb

@@ -6,7 +6,7 @@ rescue LoadError
   require 'org/slf4j/slf4j-api/1.7.21/slf4j-api-1.7.21.jar'
   require 'org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar'
   require 'log4j/log4j/1.2.17/log4j-1.2.17.jar'
-  require 'com/microsoft/azure/azure-data-lake-store-sdk/2.1.1/azure-data-lake-store-sdk-2.1.1.jar'
+  require 'com/microsoft/azure/azure-data-lake-store-sdk/2.1.5/azure-data-lake-store-sdk-2.1.5.jar'
 end
 
 if defined? Jars
@@ -14,5 +14,5 @@ if defined? Jars
   require_jar( 'org.slf4j', 'slf4j-api', '1.7.21' )
   require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.21' )
   require_jar( 'log4j', 'log4j', '1.2.17' )
-  require_jar( 'com.microsoft.azure', 'azure-data-lake-store-sdk', '2.1.1' )
+  require_jar( 'com.microsoft.azure', 'azure-data-lake-store-sdk', '2.1.5' )
 end

data/lib/logstash/outputs/adls.rb

@@ -23,6 +23,7 @@ require 'logstash-output-adls_jars.rb'
 #     adls_client_id => "00000000-0000-0000-0000-000000000000"                                  # (required)
 #     adls_client_key => "XXXXXXXXXXXXXXXXXXXXXX"                                               # (required)
 #     path => "/logstash/%{+YYYY}/%{+MM}/%{+dd}/logstash-%{+HH}-%{[@metadata][cid]}.log"        # (required)
+#     test_path => "testfile"                                                                   # (optional, default "testfile")
 #     line_separator => "\n"                                                                    # (optional, default: "\n")
 #     created_files_permission => 755                                                           # (optional, default: 755)
 #     adls_token_expire_security_margin => 300                                                  # (optional, default: 300)
@@ -58,6 +59,9 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
   # `/user/logstash/dt=%{+YYYY-MM-dd}/%{@source_host}-%{+HH}.log`
   config :path, :validate => :string, :required => true
 
+  # The path used for testing permissions in the datalake
+  config :test_path, :validate => :string, :default => "testfile"
+
   # Line separator for events written.
   config :line_separator, :validate => :string, :default => "\n"
 
@@ -96,7 +100,7 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
     def register()
 
       begin
-        @client = prepare_client(@adls_fqdn, @adls_client_id, @adls_token_endpoint, @adls_client_key)
+        @client = prepare_client(@adls_fqdn, @adls_client_id, @adls_token_endpoint, @adls_client_key,  @test_path)
       rescue => e
         logger.error("Cannot Login in ADLS. Aborting.... Exception: #{e.message}; Trace:#{e.backtrace.join("\n\t")}")
         exit 1
@@ -118,7 +122,7 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
         end
         def run
           begin
-            @parent.client = @parent.prepare_client(@parent.adls_fqdn, @parent.adls_client_id, @parent.adls_token_endpoint, @parent.adls_client_key)
+            @parent.client = @parent.prepare_client(@parent.adls_fqdn, @parent.adls_client_id, @parent.adls_token_endpoint, @parent.adls_client_key, @parent.test_path)
           rescue => e
             sleepTime = [@parent.retry_interval, @parent.max_retry_interval].min
             @parent.logger.error("ADLS Refresh OAuth Token Failed! Retrying in #{sleepTime.to_s} seconds... Exception: #{e.message}; Trace:#{e.backtrace.join("\n\t")}")         
@@ -143,7 +147,7 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
     @logger.info("Logstash ADLS output plugin is shutting down...")
   end
 
-  def prepare_client(accountFQDN, clientId, authTokenEndpoint, clientKey)
+  def prepare_client(accountFQDN, clientId, authTokenEndpoint, clientKey, testPath)
     azureToken = com.microsoft.azure.datalake.store.oauth2.AzureADAuthenticator.getTokenUsingClientCreds(authTokenEndpoint, clientId, clientKey)
 
     calendar = java.util.Calendar.getInstance()
@@ -157,7 +161,7 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
     options = com.microsoft.azure.datalake.store.ADLStoreOptions.new()
     options.setUserAgentSuffix("Logstash-ADLS-Output-Plugin")
     client.setOptions(options)
-    client.checkExists("testFile") # Test the Client to make sure it works. The return value is irrelevant. 
+    client.checkExists(testPath) # Test the Client to make sure it works. The return value is irrelevant. 
     client
   end
 
@@ -204,12 +208,11 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
       end 
     end
     @logger.debug("#{events.length.to_s} events written on ADLS in #{Time.now-timeElapsed} seconds.")
-
   end
 
   def write_data(path, data)
     begin
-      #@logger.info("Trying to write at #{path}")
+      @logger.debug("Trying to write at #{path}")
       adlsClient = @client
       
       # Try to append to already existing file, which will work most of the times.
@@ -229,11 +232,8 @@ class LogStash::Outputs::ADLS < LogStash::Outputs::Base
         createStream.close()
         @logger.debug("File #{path} created.")
       else
-         raise e  
+        raise e  
       end           
     end
-    #@logger.info("Data written to ADLS: #{data}")
-
-  end
-        
+  end 
 end

data/logstash-output-adls.gemspec

@@ -1,8 +1,9 @@
-# encoding: utf-8
+# encoding: utf-8
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-adls'
-  s.version         = '1.1.4'
+  s.version         = '1.2.0'
+  s.version         = "#{s.version}.rc#{ENV['TRAVIS_TAG'][/-rc(.*)/i,1]}" if ENV['TRAVIS_TAG'] =~ /-rc(.*)/i
   s.licenses        = ['Apache-2.0']
   s.summary         = "Plugin to write events to Azure Data Lake Store (ADLS)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -12,7 +13,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md', 'Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,15 +21,15 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
-  s.requirements << "jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.1'"
+  s.requirements << "jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.5'"
   s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.21'"
 
   s.add_development_dependency 'jar-dependencies', '~> 0.3.2'
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_development_dependency 'logstash-devutils'
 
+  s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-line'
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-codec-plain'

metadata

@@ -1,30 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-adls
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.2.0.rc3
 platform: ruby
 authors:
 - NOS Inovação
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-06 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: jar-dependencies
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
-  name: jar-dependencies
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.2
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -33,9 +34,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -45,73 +45,77 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-line
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-line
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: nosi.metadata@nos.pt
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
+- CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
 - lib/com/fasterxml/jackson/core/jackson-core/2.7.4/jackson-core-2.7.4.jar
-- lib/com/microsoft/azure/azure-data-lake-store-sdk/2.1.1/azure-data-lake-store-sdk-2.1.1.jar
+- lib/com/microsoft/azure/azure-data-lake-store-sdk/2.1.5/azure-data-lake-store-sdk-2.1.5.jar
 - lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
 - lib/logstash-output-adls_jars.rb
 - lib/logstash/outputs/adls.rb
@@ -120,7 +124,7 @@ files:
 - logstash-output-adls.gemspec
 - spec/integration/adls_spec.rb
 - spec/outputs/adls_spec.rb
-- vendor/jar-dependencies/runtime-jars/azure-data-lake-store-sdk-2.1.1.jar
+- vendor/jar-dependencies/runtime-jars/azure-data-lake-store-sdk-2.1.5.jar
 - vendor/jar-dependencies/runtime-jars/jackson-core-2.7.4.jar
 - vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.21.jar
@@ -131,7 +135,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -142,15 +146,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements:
-- jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.1'
+- jar 'com.microsoft.azure:azure-data-lake-store-sdk', '2.1.5'
 - jar 'org.slf4j:slf4j-log4j12', '1.7.21'
-rubyforge_project:
-rubygems_version: 2.6.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
 specification_version: 4
 summary: Plugin to write events to Azure Data Lake Store (ADLS)
 test_files: