logstash-output-kusto 1.0.5-java → 1.0.6-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -36
- data/CONTRIBUTORS +10 -10
- data/Gemfile +22 -22
- data/LICENSE +201 -201
- data/README.md +94 -94
- data/SECURITY.md +41 -41
- data/lib/com/fasterxml/jackson/core/jackson-annotations/2.12.5/jackson-annotations-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-core/2.12.5/jackson-core-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-databind/2.12.5/jackson-databind-2.12.5.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/3.1.1/kusto-data-3.1.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/3.2.1/kusto-data-3.2.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/3.1.1/kusto-ingest-3.1.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/3.2.1/kusto-ingest-3.2.1.jar +0 -0
- data/lib/logstash/outputs/kusto/ingestor.rb +138 -138
- data/lib/logstash/outputs/kusto/interval.rb +81 -81
- data/lib/logstash/outputs/kusto.rb +422 -422
- data/lib/logstash-output-kusto_jars.rb +12 -12
- data/lib/org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar +0 -0
- data/logstash-output-kusto.gemspec +36 -36
- data/spec/outputs/kusto/ingestor_spec.rb +121 -121
- data/spec/outputs/kusto_spec.rb +56 -56
- data/spec/spec_helpers.rb +21 -21
- metadata +21 -13
@@ -4,9 +4,7 @@ begin
|
|
4
4
|
rescue LoadError
|
5
5
|
require 'io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar'
|
6
6
|
require 'com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar'
|
7
|
-
require 'com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar'
|
8
7
|
require 'com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar'
|
9
|
-
require 'com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar'
|
10
8
|
require 'io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar'
|
11
9
|
require 'net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar'
|
12
10
|
require 'io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar'
|
@@ -15,20 +13,22 @@ rescue LoadError
|
|
15
13
|
require 'org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar'
|
16
14
|
require 'org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar'
|
17
15
|
require 'com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar'
|
16
|
+
require 'com/fasterxml/jackson/core/jackson-core/2.12.5/jackson-core-2.12.5.jar'
|
18
17
|
require 'io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar'
|
19
18
|
require 'io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar'
|
20
19
|
require 'com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar'
|
21
20
|
require 'io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar'
|
21
|
+
require 'com/fasterxml/jackson/core/jackson-annotations/2.12.5/jackson-annotations-2.12.5.jar'
|
22
22
|
require 'org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar'
|
23
23
|
require 'io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar'
|
24
24
|
require 'io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar'
|
25
|
+
require 'com/microsoft/azure/kusto/kusto-ingest/3.2.1/kusto-ingest-3.2.1.jar'
|
25
26
|
require 'io/vavr/vavr/0.10.2/vavr-0.10.2.jar'
|
26
27
|
require 'io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar'
|
27
28
|
require 'com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar'
|
28
29
|
require 'com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar'
|
29
|
-
require '
|
30
|
+
require 'com/fasterxml/jackson/core/jackson-databind/2.12.5/jackson-databind-2.12.5.jar'
|
30
31
|
require 'net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar'
|
31
|
-
require 'com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar'
|
32
32
|
require 'io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar'
|
33
33
|
require 'jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar'
|
34
34
|
require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
|
@@ -37,11 +37,10 @@ rescue LoadError
|
|
37
37
|
require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
|
38
38
|
require 'com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar'
|
39
39
|
require 'com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar'
|
40
|
+
require 'org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar'
|
40
41
|
require 'com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar'
|
41
|
-
require 'com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar'
|
42
42
|
require 'io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar'
|
43
43
|
require 'io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar'
|
44
|
-
require 'com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar'
|
45
44
|
require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
|
46
45
|
require 'org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar'
|
47
46
|
require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
|
@@ -65,6 +64,7 @@ rescue LoadError
|
|
65
64
|
require 'io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar'
|
66
65
|
require 'com/nimbusds/content-type/2.1/content-type-2.1.jar'
|
67
66
|
require 'org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar'
|
67
|
+
require 'com/microsoft/azure/kusto/kusto-data/3.2.1/kusto-data-3.2.1.jar'
|
68
68
|
require 'com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar'
|
69
69
|
require 'com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar'
|
70
70
|
require 'com/azure/azure-core/1.21.0/azure-core-1.21.0.jar'
|
@@ -78,9 +78,7 @@ end
|
|
78
78
|
if defined? Jars
|
79
79
|
require_jar 'io.netty', 'netty-resolver-dns-native-macos', 'osx-x86_64', '4.1.68.Final'
|
80
80
|
require_jar 'com.google.guava', 'guava', '24.1.1-jre'
|
81
|
-
require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.1.3'
|
82
81
|
require_jar 'com.microsoft.azure', 'azure-storage', '8.6.6'
|
83
|
-
require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.1.3'
|
84
82
|
require_jar 'io.projectreactor.netty', 'reactor-netty-http', '1.0.11'
|
85
83
|
require_jar 'net.java.dev.jna', 'jna', '5.5.0'
|
86
84
|
require_jar 'io.netty', 'netty-handler-proxy', '4.1.68.Final'
|
@@ -89,20 +87,22 @@ if defined? Jars
|
|
89
87
|
require_jar 'org.apache.commons', 'commons-lang3', '3.11'
|
90
88
|
require_jar 'org.reactivestreams', 'reactive-streams', '1.0.3'
|
91
89
|
require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.2.4'
|
90
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.12.5'
|
92
91
|
require_jar 'io.github.resilience4j', 'resilience4j-core', '1.7.1'
|
93
92
|
require_jar 'io.netty', 'netty-codec-dns', '4.1.68.Final'
|
94
93
|
require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.9.3'
|
95
94
|
require_jar 'io.netty', 'netty-transport', '4.1.68.Final'
|
95
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.12.5'
|
96
96
|
require_jar 'org.jetbrains', 'annotations', '22.0.0'
|
97
97
|
require_jar 'io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'
|
98
98
|
require_jar 'io.netty', 'netty-transport-native-epoll', 'linux-x86_64', '4.1.68.Final'
|
99
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.2.1'
|
99
100
|
require_jar 'io.vavr', 'vavr', '0.10.2'
|
100
101
|
require_jar 'io.netty', 'netty-resolver-dns', '4.1.68.Final'
|
101
102
|
require_jar 'com.azure', 'azure-identity', '1.3.7'
|
102
103
|
require_jar 'com.fasterxml.jackson.module', 'jackson-module-jaxb-annotations', '2.12.5'
|
103
|
-
require_jar '
|
104
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.12.5'
|
104
105
|
require_jar 'net.java.dev.jna', 'jna-platform', '5.6.0'
|
105
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.13.0'
|
106
106
|
require_jar 'io.netty', 'netty-resolver', '4.1.68.Final'
|
107
107
|
require_jar 'jakarta.activation', 'jakarta.activation-api', '1.2.1'
|
108
108
|
require_jar 'commons-logging', 'commons-logging', '1.2'
|
@@ -111,11 +111,10 @@ if defined? Jars
|
|
111
111
|
require_jar 'commons-codec', 'commons-codec', '1.11'
|
112
112
|
require_jar 'com.microsoft.azure', 'msal4j-persistence-extension', '1.1.0'
|
113
113
|
require_jar 'com.fasterxml.woodstox', 'woodstox-core', '6.2.4'
|
114
|
+
require_jar 'org.apache.commons', 'commons-text', '1.10.0'
|
114
115
|
require_jar 'com.github.stephenc.jcip', 'jcip-annotations', '1.0-1'
|
115
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.13.0'
|
116
116
|
require_jar 'io.netty', 'netty-transport-native-kqueue', 'osx-x86_64', '4.1.68.Final'
|
117
117
|
require_jar 'io.netty', 'netty-handler', '4.1.68.Final'
|
118
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.13.0'
|
119
118
|
require_jar 'com.google.code.findbugs', 'jsr305', '1.3.9'
|
120
119
|
require_jar 'org.checkerframework', 'checker-compat-qual', '2.0.0'
|
121
120
|
require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
|
@@ -139,6 +138,7 @@ if defined? Jars
|
|
139
138
|
require_jar 'io.vavr', 'vavr-match', '0.10.2'
|
140
139
|
require_jar 'com.nimbusds', 'content-type', '2.1'
|
141
140
|
require_jar 'org.slf4j', 'slf4j-api', '1.8.0-beta4'
|
141
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.2.1'
|
142
142
|
require_jar 'com.microsoft.azure', 'msal4j', '1.11.0'
|
143
143
|
require_jar 'com.nimbusds', 'lang-tag', '1.5'
|
144
144
|
require_jar 'com.azure', 'azure-core', '1.21.0'
|
Binary file
|
@@ -1,36 +1,36 @@
|
|
1
|
-
Gem::Specification.new do |s|
|
2
|
-
s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
|
3
|
-
s.version = '1.0.
|
4
|
-
s.licenses = ['Apache-2.0']
|
5
|
-
s.summary = 'Writes events to Azure Data Explorer (Kusto)'
|
6
|
-
s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
|
7
|
-
s.homepage = 'https://github.com/Azure/logstash-output-kusto'
|
8
|
-
s.authors = ['Tamir Kamara', 'Asaf Mahlev']
|
9
|
-
s.email = 'nugetkusto@microsoft.com'
|
10
|
-
s.require_paths = ['lib']
|
11
|
-
s.platform = 'java'
|
12
|
-
|
13
|
-
# Files
|
14
|
-
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
|
15
|
-
|
16
|
-
# Tests
|
17
|
-
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
-
|
19
|
-
# Special flag to let us know this is actually a logstash plugin
|
20
|
-
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
21
|
-
|
22
|
-
# Gem dependencies
|
23
|
-
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
|
24
|
-
s.add_runtime_dependency 'logstash-codec-json_lines'
|
25
|
-
s.add_runtime_dependency 'logstash-codec-line'
|
26
|
-
|
27
|
-
s.add_development_dependency 'logstash-devutils'
|
28
|
-
s.add_development_dependency 'flores'
|
29
|
-
s.add_development_dependency 'logstash-input-generator'
|
30
|
-
s.add_development_dependency 'ruby-maven', '~> 3.3.11'
|
31
|
-
s.add_development_dependency 'rspec_junit_formatter'
|
32
|
-
|
33
|
-
# Jar dependencies
|
34
|
-
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1
|
35
|
-
s.add_runtime_dependency 'jar-dependencies'
|
36
|
-
end
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
|
3
|
+
s.version = '1.0.6'
|
4
|
+
s.licenses = ['Apache-2.0']
|
5
|
+
s.summary = 'Writes events to Azure Data Explorer (Kusto)'
|
6
|
+
s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
|
7
|
+
s.homepage = 'https://github.com/Azure/logstash-output-kusto'
|
8
|
+
s.authors = ['Tamir Kamara', 'Asaf Mahlev']
|
9
|
+
s.email = 'nugetkusto@microsoft.com'
|
10
|
+
s.require_paths = ['lib']
|
11
|
+
s.platform = 'java'
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
|
24
|
+
s.add_runtime_dependency 'logstash-codec-json_lines'
|
25
|
+
s.add_runtime_dependency 'logstash-codec-line'
|
26
|
+
|
27
|
+
s.add_development_dependency 'logstash-devutils'
|
28
|
+
s.add_development_dependency 'flores'
|
29
|
+
s.add_development_dependency 'logstash-input-generator'
|
30
|
+
s.add_development_dependency 'ruby-maven', '~> 3.3.11'
|
31
|
+
s.add_development_dependency 'rspec_junit_formatter'
|
32
|
+
|
33
|
+
# Jar dependencies
|
34
|
+
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.2.1'"
|
35
|
+
s.add_runtime_dependency 'jar-dependencies'
|
36
|
+
end
|
@@ -1,121 +1,121 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require_relative "../../spec_helpers.rb"
|
3
|
-
require 'logstash/outputs/kusto'
|
4
|
-
require 'logstash/outputs/kusto/ingestor'
|
5
|
-
|
6
|
-
describe LogStash::Outputs::Kusto::Ingestor do
|
7
|
-
|
8
|
-
let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
|
9
|
-
let(:app_id) { "myid" }
|
10
|
-
let(:app_key) { LogStash::Util::Password.new("mykey") }
|
11
|
-
let(:app_tenant) { "mytenant" }
|
12
|
-
let(:database) { "mydatabase" }
|
13
|
-
let(:table) { "mytable" }
|
14
|
-
let(:proxy_host) { "localhost" }
|
15
|
-
let(:proxy_port) { 80 }
|
16
|
-
let(:proxy_protocol) { "http" }
|
17
|
-
let(:json_mapping) { "mymapping" }
|
18
|
-
let(:delete_local) { false }
|
19
|
-
let(:logger) { spy('logger') }
|
20
|
-
|
21
|
-
describe '#initialize' do
|
22
|
-
|
23
|
-
it 'does not throw an error when initializing' do
|
24
|
-
# note that this will cause an internal error since connection is being tried.
|
25
|
-
# however we still want to test that all the java stuff is working as expected
|
26
|
-
expect {
|
27
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
|
28
|
-
ingestor.stop
|
29
|
-
}.not_to raise_error
|
30
|
-
end
|
31
|
-
|
32
|
-
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
33
|
-
|
34
|
-
context 'doesnt allow database to have some dynamic part' do
|
35
|
-
dynamic_name_array.each do |test_database|
|
36
|
-
it "with database: #{test_database}" do
|
37
|
-
expect {
|
38
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
39
|
-
ingestor.stop
|
40
|
-
}.to raise_error(LogStash::ConfigurationError)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
context 'doesnt allow table to have some dynamic part' do
|
46
|
-
dynamic_name_array.each do |test_table|
|
47
|
-
it "with database: #{test_table}" do
|
48
|
-
expect {
|
49
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
50
|
-
ingestor.stop
|
51
|
-
}.to raise_error(LogStash::ConfigurationError)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
context 'doesnt allow mapping to have some dynamic part' do
|
57
|
-
dynamic_name_array.each do |json_mapping|
|
58
|
-
it "with database: #{json_mapping}" do
|
59
|
-
expect {
|
60
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
61
|
-
ingestor.stop
|
62
|
-
}.to raise_error(LogStash::ConfigurationError)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
context 'proxy protocol has to be http or https' do
|
68
|
-
it "with proxy protocol: socks" do
|
69
|
-
expect {
|
70
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
|
71
|
-
ingestor.stop
|
72
|
-
}.to raise_error(LogStash::ConfigurationError)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
# describe 'receiving events' do
|
79
|
-
|
80
|
-
# context 'with non-zero flush interval' do
|
81
|
-
# let(:temporary_output_file) { Stud::Temporary.pathname }
|
82
|
-
|
83
|
-
# let(:event_count) { 100 }
|
84
|
-
# let(:flush_interval) { 5 }
|
85
|
-
|
86
|
-
# let(:events) do
|
87
|
-
# event_count.times.map do |idx|
|
88
|
-
# LogStash::Event.new('subject' => idx)
|
89
|
-
# end
|
90
|
-
# end
|
91
|
-
|
92
|
-
# let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
|
93
|
-
|
94
|
-
# before(:each) { output.register }
|
95
|
-
|
96
|
-
# after(:each) do
|
97
|
-
# output.close
|
98
|
-
# File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
|
99
|
-
# File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
|
100
|
-
# end
|
101
|
-
|
102
|
-
# it 'eventually flushes without receiving additional events' do
|
103
|
-
# output.multi_receive_encoded(events)
|
104
|
-
|
105
|
-
# # events should not all be flushed just yet...
|
106
|
-
# expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
|
107
|
-
# contents && contents.lines.count < event_count
|
108
|
-
# end
|
109
|
-
|
110
|
-
# # wait for the flusher to run...
|
111
|
-
# sleep(flush_interval + 1)
|
112
|
-
|
113
|
-
# # events should all be flushed
|
114
|
-
# expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
|
115
|
-
# contents && contents.lines.count == event_count
|
116
|
-
# end
|
117
|
-
# end
|
118
|
-
# end
|
119
|
-
|
120
|
-
# end
|
121
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../../spec_helpers.rb"
|
3
|
+
require 'logstash/outputs/kusto'
|
4
|
+
require 'logstash/outputs/kusto/ingestor'
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Kusto::Ingestor do
|
7
|
+
|
8
|
+
let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
|
9
|
+
let(:app_id) { "myid" }
|
10
|
+
let(:app_key) { LogStash::Util::Password.new("mykey") }
|
11
|
+
let(:app_tenant) { "mytenant" }
|
12
|
+
let(:database) { "mydatabase" }
|
13
|
+
let(:table) { "mytable" }
|
14
|
+
let(:proxy_host) { "localhost" }
|
15
|
+
let(:proxy_port) { 80 }
|
16
|
+
let(:proxy_protocol) { "http" }
|
17
|
+
let(:json_mapping) { "mymapping" }
|
18
|
+
let(:delete_local) { false }
|
19
|
+
let(:logger) { spy('logger') }
|
20
|
+
|
21
|
+
describe '#initialize' do
|
22
|
+
|
23
|
+
it 'does not throw an error when initializing' do
|
24
|
+
# note that this will cause an internal error since connection is being tried.
|
25
|
+
# however we still want to test that all the java stuff is working as expected
|
26
|
+
expect {
|
27
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
|
28
|
+
ingestor.stop
|
29
|
+
}.not_to raise_error
|
30
|
+
end
|
31
|
+
|
32
|
+
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
33
|
+
|
34
|
+
context 'doesnt allow database to have some dynamic part' do
|
35
|
+
dynamic_name_array.each do |test_database|
|
36
|
+
it "with database: #{test_database}" do
|
37
|
+
expect {
|
38
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
39
|
+
ingestor.stop
|
40
|
+
}.to raise_error(LogStash::ConfigurationError)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'doesnt allow table to have some dynamic part' do
|
46
|
+
dynamic_name_array.each do |test_table|
|
47
|
+
it "with database: #{test_table}" do
|
48
|
+
expect {
|
49
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
50
|
+
ingestor.stop
|
51
|
+
}.to raise_error(LogStash::ConfigurationError)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'doesnt allow mapping to have some dynamic part' do
|
57
|
+
dynamic_name_array.each do |json_mapping|
|
58
|
+
it "with database: #{json_mapping}" do
|
59
|
+
expect {
|
60
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
61
|
+
ingestor.stop
|
62
|
+
}.to raise_error(LogStash::ConfigurationError)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'proxy protocol has to be http or https' do
|
68
|
+
it "with proxy protocol: socks" do
|
69
|
+
expect {
|
70
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
|
71
|
+
ingestor.stop
|
72
|
+
}.to raise_error(LogStash::ConfigurationError)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
# describe 'receiving events' do
|
79
|
+
|
80
|
+
# context 'with non-zero flush interval' do
|
81
|
+
# let(:temporary_output_file) { Stud::Temporary.pathname }
|
82
|
+
|
83
|
+
# let(:event_count) { 100 }
|
84
|
+
# let(:flush_interval) { 5 }
|
85
|
+
|
86
|
+
# let(:events) do
|
87
|
+
# event_count.times.map do |idx|
|
88
|
+
# LogStash::Event.new('subject' => idx)
|
89
|
+
# end
|
90
|
+
# end
|
91
|
+
|
92
|
+
# let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
|
93
|
+
|
94
|
+
# before(:each) { output.register }
|
95
|
+
|
96
|
+
# after(:each) do
|
97
|
+
# output.close
|
98
|
+
# File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
|
99
|
+
# File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
|
100
|
+
# end
|
101
|
+
|
102
|
+
# it 'eventually flushes without receiving additional events' do
|
103
|
+
# output.multi_receive_encoded(events)
|
104
|
+
|
105
|
+
# # events should not all be flushed just yet...
|
106
|
+
# expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
|
107
|
+
# contents && contents.lines.count < event_count
|
108
|
+
# end
|
109
|
+
|
110
|
+
# # wait for the flusher to run...
|
111
|
+
# sleep(flush_interval + 1)
|
112
|
+
|
113
|
+
# # events should all be flushed
|
114
|
+
# expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
|
115
|
+
# contents && contents.lines.count == event_count
|
116
|
+
# end
|
117
|
+
# end
|
118
|
+
# end
|
119
|
+
|
120
|
+
# end
|
121
|
+
end
|
data/spec/outputs/kusto_spec.rb
CHANGED
@@ -1,56 +1,56 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'logstash/outputs/kusto'
|
3
|
-
require 'logstash/codecs/plain'
|
4
|
-
require 'logstash/event'
|
5
|
-
|
6
|
-
describe LogStash::Outputs::Kusto do
|
7
|
-
|
8
|
-
let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
|
9
|
-
"ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
|
10
|
-
"app_id" => "myid",
|
11
|
-
"app_key" => "mykey",
|
12
|
-
"app_tenant" => "mytenant",
|
13
|
-
"database" => "mydatabase",
|
14
|
-
"table" => "mytable",
|
15
|
-
"json_mapping" => "mymapping",
|
16
|
-
"proxy_host" => "localhost",
|
17
|
-
"proxy_port" => 3128,
|
18
|
-
"proxy_protocol" => "https"
|
19
|
-
} }
|
20
|
-
|
21
|
-
describe '#register' do
|
22
|
-
|
23
|
-
it 'doesnt allow the path to start with a dynamic string' do
|
24
|
-
kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
|
25
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
26
|
-
kusto.close
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'path must include a dynamic string to allow file rotation' do
|
30
|
-
kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
|
31
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
32
|
-
kusto.close
|
33
|
-
end
|
34
|
-
|
35
|
-
|
36
|
-
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
37
|
-
|
38
|
-
context 'doesnt allow the root directory to have some dynamic part' do
|
39
|
-
dynamic_name_array.each do |test_path|
|
40
|
-
it "with path: #{test_path}" do
|
41
|
-
kusto = described_class.new(options.merge( {'path' => test_path} ))
|
42
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
43
|
-
kusto.close
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
it 'allow to have dynamic part after the file root' do
|
49
|
-
kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
|
50
|
-
expect { kusto.register }.not_to raise_error
|
51
|
-
kusto.close
|
52
|
-
end
|
53
|
-
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'logstash/outputs/kusto'
|
3
|
+
require 'logstash/codecs/plain'
|
4
|
+
require 'logstash/event'
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Kusto do
|
7
|
+
|
8
|
+
let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
|
9
|
+
"ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
|
10
|
+
"app_id" => "myid",
|
11
|
+
"app_key" => "mykey",
|
12
|
+
"app_tenant" => "mytenant",
|
13
|
+
"database" => "mydatabase",
|
14
|
+
"table" => "mytable",
|
15
|
+
"json_mapping" => "mymapping",
|
16
|
+
"proxy_host" => "localhost",
|
17
|
+
"proxy_port" => 3128,
|
18
|
+
"proxy_protocol" => "https"
|
19
|
+
} }
|
20
|
+
|
21
|
+
describe '#register' do
|
22
|
+
|
23
|
+
it 'doesnt allow the path to start with a dynamic string' do
|
24
|
+
kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
|
25
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
26
|
+
kusto.close
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'path must include a dynamic string to allow file rotation' do
|
30
|
+
kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
|
31
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
32
|
+
kusto.close
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
37
|
+
|
38
|
+
context 'doesnt allow the root directory to have some dynamic part' do
|
39
|
+
dynamic_name_array.each do |test_path|
|
40
|
+
it "with path: #{test_path}" do
|
41
|
+
kusto = described_class.new(options.merge( {'path' => test_path} ))
|
42
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
43
|
+
kusto.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'allow to have dynamic part after the file root' do
|
49
|
+
kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
|
50
|
+
expect { kusto.register }.not_to raise_error
|
51
|
+
kusto.close
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
data/spec/spec_helpers.rb
CHANGED
@@ -1,21 +1,21 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/logging/logger"
|
4
|
-
|
5
|
-
LogStash::Logging::Logger::configure_logging("debug")
|
6
|
-
|
7
|
-
RSpec.configure do |config|
|
8
|
-
# register around filter that captures stdout and stderr
|
9
|
-
config.around(:each) do |example|
|
10
|
-
$stdout = StringIO.new
|
11
|
-
$stderr = StringIO.new
|
12
|
-
|
13
|
-
example.run
|
14
|
-
|
15
|
-
example.metadata[:stdout] = $stdout.string
|
16
|
-
example.metadata[:stderr] = $stderr.string
|
17
|
-
|
18
|
-
$stdout = STDOUT
|
19
|
-
$stderr = STDERR
|
20
|
-
end
|
21
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/logging/logger"
|
4
|
+
|
5
|
+
LogStash::Logging::Logger::configure_logging("debug")
|
6
|
+
|
7
|
+
RSpec.configure do |config|
|
8
|
+
# register around filter that captures stdout and stderr
|
9
|
+
config.around(:each) do |example|
|
10
|
+
$stdout = StringIO.new
|
11
|
+
$stderr = StringIO.new
|
12
|
+
|
13
|
+
example.run
|
14
|
+
|
15
|
+
example.metadata[:stdout] = $stdout.string
|
16
|
+
example.metadata[:stderr] = $stderr.string
|
17
|
+
|
18
|
+
$stdout = STDOUT
|
19
|
+
$stderr = STDERR
|
20
|
+
end
|
21
|
+
end
|