logstash-output-kusto 1.0.0-java

data/lib/logstash/outputs/kusto/interval.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+require 'logstash/outputs/base'
+require 'logstash/namespace'
+require 'logstash/errors'
+
+class LogStash::Outputs::Kusto < LogStash::Outputs::Base
+  ##
+  # Bare-bones utility for running a block of code at an interval.
+  #
+  class Interval
+    ##
+    # Initializes a new Interval with the given arguments and starts it
+    # before returning it.
+    #
+    # @param interval [Integer] (see: Interval#initialize)
+    # @param procsy [#call] (see: Interval#initialize)
+    #
+    # @return [Interval]
+    #
+    def self.start(interval, procsy)
+      new(interval, procsy).tap(&:start)
+    end
+
+    ##
+    # @param interval [Integer]: time in seconds to wait between calling the given proc
+    # @param procsy [#call]: proc or lambda to call periodically; must not raise exceptions.
+    def initialize(interval, procsy)
+      @interval = interval
+      @procsy = procsy
+
+      # Mutex, ConditionVariable, etc.
+      @mutex = Mutex.new
+      @sleeper = ConditionVariable.new
+    end
+
+    ##
+    # Starts the interval, or returns if it has already been started.
+    #
+    # @return [void]
+    def start
+      @mutex.synchronize do
+        return if @thread && @thread.alive?
+
+        @thread = Thread.new { run }
+      end
+    end
+
+    ##
+    # Stop the interval.
+    # Does not interrupt if execution is in-progress.
+    def stop
+      @mutex.synchronize do
+        @stopped = true
+      end
+
+      @thread && @thread.join
+    end
+
+    ##
+    # @return [Boolean]
+    def alive?
+      @thread && @thread.alive?
+    end
+
+    private
+
+    def run
+      @mutex.synchronize do
+        loop do
+          @sleeper.wait(@mutex, @interval)
+          break if @stopped
+
+          @procsy.call
+        end
+      end
+    ensure
+      @sleeper.broadcast
+    end
+  end
+end

data/logstash-output-kusto.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
+  s.version       = '1.0.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Writes events to Azure Data Explorer (Kusto)'
+  s.description   = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
+  s.homepage      = 'https://github.com/Azure/logstash-output-kusto'
+  s.authors       = ['Tamir Kamara', 'Asaf Mahlev']
+  s.email         = 'nugetkusto@microsoft.com'
+  s.require_paths = ['lib']
+  s.platform = 'java'
+
+  # Files
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
+  s.add_runtime_dependency 'logstash-codec-line'
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'flores'
+  s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'ruby-maven', '~> 3.3.11'
+
+  # Jar dependencies
+  s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2"
+  s.add_runtime_dependency 'jar-dependencies'
+end

data/spec/outputs/kusto/ingestor_spec.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+require_relative "../../spec_helpers.rb"
+require 'logstash/outputs/kusto'
+require 'logstash/outputs/kusto/ingestor'
+
+describe LogStash::Outputs::Kusto::Ingestor do
+
+  let(:ingest_url) { "mycluster" }
+  let(:app_id) { "myid" }
+  let(:app_key) { LogStash::Util::Password.new("mykey") }
+  let(:app_tenant) { "mytenant" }
+  let(:database) { "mydatabase" }
+  let(:table) { "mytable" }
+  let(:json_mapping) { "mymapping" }
+  let(:delete_local) { false }
+  let(:logger) { spy('logger') }
+
+  describe '#initialize' do
+
+    it 'does not throw an error when initializing' do
+      # note that this will cause an internal error since connection is being tried.
+      # however we still want to test that all the java stuff is working as expected
+      expect { 
+        ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger)
+        ingestor.stop
+      }.not_to raise_error
+    end
+    
+    dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
+
+    context 'doesnt allow database to have some dynamic part' do
+      dynamic_name_array.each do |test_database|
+        it "with database: #{test_database}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, logger)
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+    context 'doesnt allow table to have some dynamic part' do
+      dynamic_name_array.each do |test_table|
+        it "with database: #{test_table}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, logger)
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+    context 'doesnt allow mapping to have some dynamic part' do
+      dynamic_name_array.each do |test_json_mapping|
+        it "with database: #{test_json_mapping}" do
+          expect {
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, test_json_mapping, delete_local, logger)
+            ingestor.stop
+          }.to raise_error(LogStash::ConfigurationError)          
+        end
+      end
+    end
+
+  end
+
+  # describe 'receiving events' do
+
+  #   context 'with non-zero flush interval' do
+  #     let(:temporary_output_file) { Stud::Temporary.pathname }
+
+  #     let(:event_count) { 100 }
+  #     let(:flush_interval) { 5 }
+
+  #     let(:events) do
+  #       event_count.times.map do |idx|
+  #         LogStash::Event.new('subject' => idx)
+  #       end
+  #     end
+
+  #     let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
+
+  #     before(:each) { output.register }
+      
+  #     after(:each) do
+  #       output.close
+  #       File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
+  #       File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
+  #     end
+
+  #     it 'eventually flushes without receiving additional events' do
+  #       output.multi_receive_encoded(events)
+
+  #       # events should not all be flushed just yet...
+  #       expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
+  #         contents && contents.lines.count < event_count
+  #       end
+
+  #       # wait for the flusher to run...
+  #       sleep(flush_interval + 1)
+
+  #       # events should all be flushed
+  #       expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
+  #         contents && contents.lines.count == event_count
+  #       end
+  #     end
+  #   end
+
+  # end
+end

data/spec/outputs/kusto_spec.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require 'logstash/devutils/rspec/spec_helper'
+require 'logstash/outputs/kusto'
+require 'logstash/codecs/plain'
+require 'logstash/event'
+
+describe LogStash::Outputs::Kusto do
+
+  let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
+    "ingest_url" => "mycluster",
+    "app_id" => "myid",
+    "app_key" => "mykey",
+    "app_tenant" => "mytenant", 
+    "database" => "mydatabase",
+    "table" => "mytable",
+    "json_mapping" => "mymapping"
+  } }
+
+  describe '#register' do
+
+    it 'doesnt allow the path to start with a dynamic string' do
+      kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
+      expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+      kusto.close
+    end
+
+    it 'path must include a dynamic string to allow file rotation' do
+      kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
+      expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+      kusto.close
+    end
+
+
+    dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
+
+    context 'doesnt allow the root directory to have some dynamic part' do
+      dynamic_name_array.each do |test_path|
+         it "with path: #{test_path}" do
+           kusto = described_class.new(options.merge( {'path' => test_path} ))
+           expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
+           kusto.close
+         end
+       end
+    end
+
+    it 'allow to have dynamic part after the file root' do
+      kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
+      expect { kusto.register }.not_to raise_error
+      kusto.close
+    end
+
+  end
+  
+end

data/spec/spec_helpers.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/logging/logger"
+
+LogStash::Logging::Logger::configure_logging("debug")
+
+RSpec.configure do |config|
+  # register around filter that captures stdout and stderr
+  config.around(:each) do |example|
+    $stdout = StringIO.new
+    $stderr = StringIO.new
+
+    example.run
+
+    example.metadata[:stdout] = $stdout.string
+    example.metadata[:stderr] = $stderr.string
+
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+end

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-kusto
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: java
+authors:
+- Tamir Kamara
+- Asaf Mahlev
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-12-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: flores
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-input-generator
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.11
+  name: ruby-maven
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.11
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: jar-dependencies
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This is a logstash output plugin used to write events to an Azure Data
+  Explorer (a.k.a Kusto)
+email: nugetkusto@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar
+- lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar
+- lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar
+- lib/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar
+- lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar
+- lib/com/google/guava/guava/20.0/guava-20.0.jar
+- lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar
+- lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar
+- lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar
+- lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar
+- lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar
+- lib/com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar
+- lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar
+- lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar
+- lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar
+- lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar
+- lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar
+- lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
+- lib/javax/activation/activation/1.1/activation-1.1.jar
+- lib/logstash-output-kusto_jars.rb
+- lib/logstash/outputs/kusto.rb
+- lib/logstash/outputs/kusto/ingestor.rb
+- lib/logstash/outputs/kusto/interval.rb
+- lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar
+- lib/net/minidev/json-smart/2.3/json-smart-2.3.jar
+- lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar
+- lib/org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar
+- lib/org/apache/httpcomponents/httpcore/4.4.11/httpcore-4.4.11.jar
+- lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar
+- lib/org/json/json/20190722/json-20190722.jar
+- lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar
+- lib/org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar
+- logstash-output-kusto.gemspec
+- spec/outputs/kusto/ingestor_spec.rb
+- spec/outputs/kusto_spec.rb
+- spec/spec_helpers.rb
+homepage: https://github.com/Azure/logstash-output-kusto
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2
+rubygems_version: 3.0.6
+signing_key:
+specification_version: 4
+summary: Writes events to Azure Data Explorer (Kusto)
+test_files:
+- spec/outputs/kusto/ingestor_spec.rb
+- spec/outputs/kusto_spec.rb
+- spec/spec_helpers.rb