logstash-output-kusto 1.0.5-java → 1.0.6-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -36
- data/CONTRIBUTORS +10 -10
- data/Gemfile +22 -22
- data/LICENSE +201 -201
- data/README.md +94 -94
- data/SECURITY.md +41 -41
- data/lib/com/fasterxml/jackson/core/jackson-annotations/2.12.5/jackson-annotations-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-core/2.12.5/jackson-core-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-databind/2.12.5/jackson-databind-2.12.5.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/3.1.1/kusto-data-3.1.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/3.2.1/kusto-data-3.2.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/3.1.1/kusto-ingest-3.1.1.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/3.2.1/kusto-ingest-3.2.1.jar +0 -0
- data/lib/logstash/outputs/kusto/ingestor.rb +138 -138
- data/lib/logstash/outputs/kusto/interval.rb +81 -81
- data/lib/logstash/outputs/kusto.rb +422 -422
- data/lib/logstash-output-kusto_jars.rb +12 -12
- data/lib/org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar +0 -0
- data/logstash-output-kusto.gemspec +36 -36
- data/spec/outputs/kusto/ingestor_spec.rb +121 -121
- data/spec/outputs/kusto_spec.rb +56 -56
- data/spec/spec_helpers.rb +21 -21
- metadata +21 -13
@@ -4,9 +4,7 @@ begin
|
|
4
4
|
rescue LoadError
|
5
5
|
require 'io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar'
|
6
6
|
require 'com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar'
|
7
|
-
require 'com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar'
|
8
7
|
require 'com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar'
|
9
|
-
require 'com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar'
|
10
8
|
require 'io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar'
|
11
9
|
require 'net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar'
|
12
10
|
require 'io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar'
|
@@ -15,20 +13,22 @@ rescue LoadError
|
|
15
13
|
require 'org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar'
|
16
14
|
require 'org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar'
|
17
15
|
require 'com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar'
|
16
|
+
require 'com/fasterxml/jackson/core/jackson-core/2.12.5/jackson-core-2.12.5.jar'
|
18
17
|
require 'io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar'
|
19
18
|
require 'io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar'
|
20
19
|
require 'com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar'
|
21
20
|
require 'io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar'
|
21
|
+
require 'com/fasterxml/jackson/core/jackson-annotations/2.12.5/jackson-annotations-2.12.5.jar'
|
22
22
|
require 'org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar'
|
23
23
|
require 'io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar'
|
24
24
|
require 'io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar'
|
25
|
+
require 'com/microsoft/azure/kusto/kusto-ingest/3.2.1/kusto-ingest-3.2.1.jar'
|
25
26
|
require 'io/vavr/vavr/0.10.2/vavr-0.10.2.jar'
|
26
27
|
require 'io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar'
|
27
28
|
require 'com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar'
|
28
29
|
require 'com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar'
|
29
|
-
require '
|
30
|
+
require 'com/fasterxml/jackson/core/jackson-databind/2.12.5/jackson-databind-2.12.5.jar'
|
30
31
|
require 'net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar'
|
31
|
-
require 'com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar'
|
32
32
|
require 'io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar'
|
33
33
|
require 'jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar'
|
34
34
|
require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
|
@@ -37,11 +37,10 @@ rescue LoadError
|
|
37
37
|
require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
|
38
38
|
require 'com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar'
|
39
39
|
require 'com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar'
|
40
|
+
require 'org/apache/commons/commons-text/1.10.0/commons-text-1.10.0.jar'
|
40
41
|
require 'com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar'
|
41
|
-
require 'com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar'
|
42
42
|
require 'io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar'
|
43
43
|
require 'io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar'
|
44
|
-
require 'com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar'
|
45
44
|
require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
|
46
45
|
require 'org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar'
|
47
46
|
require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
|
@@ -65,6 +64,7 @@ rescue LoadError
|
|
65
64
|
require 'io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar'
|
66
65
|
require 'com/nimbusds/content-type/2.1/content-type-2.1.jar'
|
67
66
|
require 'org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar'
|
67
|
+
require 'com/microsoft/azure/kusto/kusto-data/3.2.1/kusto-data-3.2.1.jar'
|
68
68
|
require 'com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar'
|
69
69
|
require 'com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar'
|
70
70
|
require 'com/azure/azure-core/1.21.0/azure-core-1.21.0.jar'
|
@@ -78,9 +78,7 @@ end
|
|
78
78
|
if defined? Jars
|
79
79
|
require_jar 'io.netty', 'netty-resolver-dns-native-macos', 'osx-x86_64', '4.1.68.Final'
|
80
80
|
require_jar 'com.google.guava', 'guava', '24.1.1-jre'
|
81
|
-
require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.1.3'
|
82
81
|
require_jar 'com.microsoft.azure', 'azure-storage', '8.6.6'
|
83
|
-
require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.1.3'
|
84
82
|
require_jar 'io.projectreactor.netty', 'reactor-netty-http', '1.0.11'
|
85
83
|
require_jar 'net.java.dev.jna', 'jna', '5.5.0'
|
86
84
|
require_jar 'io.netty', 'netty-handler-proxy', '4.1.68.Final'
|
@@ -89,20 +87,22 @@ if defined? Jars
|
|
89
87
|
require_jar 'org.apache.commons', 'commons-lang3', '3.11'
|
90
88
|
require_jar 'org.reactivestreams', 'reactive-streams', '1.0.3'
|
91
89
|
require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.2.4'
|
90
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.12.5'
|
92
91
|
require_jar 'io.github.resilience4j', 'resilience4j-core', '1.7.1'
|
93
92
|
require_jar 'io.netty', 'netty-codec-dns', '4.1.68.Final'
|
94
93
|
require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.9.3'
|
95
94
|
require_jar 'io.netty', 'netty-transport', '4.1.68.Final'
|
95
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.12.5'
|
96
96
|
require_jar 'org.jetbrains', 'annotations', '22.0.0'
|
97
97
|
require_jar 'io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'
|
98
98
|
require_jar 'io.netty', 'netty-transport-native-epoll', 'linux-x86_64', '4.1.68.Final'
|
99
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.2.1'
|
99
100
|
require_jar 'io.vavr', 'vavr', '0.10.2'
|
100
101
|
require_jar 'io.netty', 'netty-resolver-dns', '4.1.68.Final'
|
101
102
|
require_jar 'com.azure', 'azure-identity', '1.3.7'
|
102
103
|
require_jar 'com.fasterxml.jackson.module', 'jackson-module-jaxb-annotations', '2.12.5'
|
103
|
-
require_jar '
|
104
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.12.5'
|
104
105
|
require_jar 'net.java.dev.jna', 'jna-platform', '5.6.0'
|
105
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.13.0'
|
106
106
|
require_jar 'io.netty', 'netty-resolver', '4.1.68.Final'
|
107
107
|
require_jar 'jakarta.activation', 'jakarta.activation-api', '1.2.1'
|
108
108
|
require_jar 'commons-logging', 'commons-logging', '1.2'
|
@@ -111,11 +111,10 @@ if defined? Jars
|
|
111
111
|
require_jar 'commons-codec', 'commons-codec', '1.11'
|
112
112
|
require_jar 'com.microsoft.azure', 'msal4j-persistence-extension', '1.1.0'
|
113
113
|
require_jar 'com.fasterxml.woodstox', 'woodstox-core', '6.2.4'
|
114
|
+
require_jar 'org.apache.commons', 'commons-text', '1.10.0'
|
114
115
|
require_jar 'com.github.stephenc.jcip', 'jcip-annotations', '1.0-1'
|
115
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.13.0'
|
116
116
|
require_jar 'io.netty', 'netty-transport-native-kqueue', 'osx-x86_64', '4.1.68.Final'
|
117
117
|
require_jar 'io.netty', 'netty-handler', '4.1.68.Final'
|
118
|
-
require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.13.0'
|
119
118
|
require_jar 'com.google.code.findbugs', 'jsr305', '1.3.9'
|
120
119
|
require_jar 'org.checkerframework', 'checker-compat-qual', '2.0.0'
|
121
120
|
require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
|
@@ -139,6 +138,7 @@ if defined? Jars
|
|
139
138
|
require_jar 'io.vavr', 'vavr-match', '0.10.2'
|
140
139
|
require_jar 'com.nimbusds', 'content-type', '2.1'
|
141
140
|
require_jar 'org.slf4j', 'slf4j-api', '1.8.0-beta4'
|
141
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.2.1'
|
142
142
|
require_jar 'com.microsoft.azure', 'msal4j', '1.11.0'
|
143
143
|
require_jar 'com.nimbusds', 'lang-tag', '1.5'
|
144
144
|
require_jar 'com.azure', 'azure-core', '1.21.0'
|
Binary file
|
@@ -1,36 +1,36 @@
|
|
1
|
-
Gem::Specification.new do |s|
|
2
|
-
s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
|
3
|
-
s.version = '1.0.
|
4
|
-
s.licenses = ['Apache-2.0']
|
5
|
-
s.summary = 'Writes events to Azure Data Explorer (Kusto)'
|
6
|
-
s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
|
7
|
-
s.homepage = 'https://github.com/Azure/logstash-output-kusto'
|
8
|
-
s.authors = ['Tamir Kamara', 'Asaf Mahlev']
|
9
|
-
s.email = 'nugetkusto@microsoft.com'
|
10
|
-
s.require_paths = ['lib']
|
11
|
-
s.platform = 'java'
|
12
|
-
|
13
|
-
# Files
|
14
|
-
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
|
15
|
-
|
16
|
-
# Tests
|
17
|
-
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
-
|
19
|
-
# Special flag to let us know this is actually a logstash plugin
|
20
|
-
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
21
|
-
|
22
|
-
# Gem dependencies
|
23
|
-
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
|
24
|
-
s.add_runtime_dependency 'logstash-codec-json_lines'
|
25
|
-
s.add_runtime_dependency 'logstash-codec-line'
|
26
|
-
|
27
|
-
s.add_development_dependency 'logstash-devutils'
|
28
|
-
s.add_development_dependency 'flores'
|
29
|
-
s.add_development_dependency 'logstash-input-generator'
|
30
|
-
s.add_development_dependency 'ruby-maven', '~> 3.3.11'
|
31
|
-
s.add_development_dependency 'rspec_junit_formatter'
|
32
|
-
|
33
|
-
# Jar dependencies
|
34
|
-
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1
|
35
|
-
s.add_runtime_dependency 'jar-dependencies'
|
36
|
-
end
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
|
3
|
+
s.version = '1.0.6'
|
4
|
+
s.licenses = ['Apache-2.0']
|
5
|
+
s.summary = 'Writes events to Azure Data Explorer (Kusto)'
|
6
|
+
s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
|
7
|
+
s.homepage = 'https://github.com/Azure/logstash-output-kusto'
|
8
|
+
s.authors = ['Tamir Kamara', 'Asaf Mahlev']
|
9
|
+
s.email = 'nugetkusto@microsoft.com'
|
10
|
+
s.require_paths = ['lib']
|
11
|
+
s.platform = 'java'
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
|
15
|
+
|
16
|
+
# Tests
|
17
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
18
|
+
|
19
|
+
# Special flag to let us know this is actually a logstash plugin
|
20
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
21
|
+
|
22
|
+
# Gem dependencies
|
23
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
|
24
|
+
s.add_runtime_dependency 'logstash-codec-json_lines'
|
25
|
+
s.add_runtime_dependency 'logstash-codec-line'
|
26
|
+
|
27
|
+
s.add_development_dependency 'logstash-devutils'
|
28
|
+
s.add_development_dependency 'flores'
|
29
|
+
s.add_development_dependency 'logstash-input-generator'
|
30
|
+
s.add_development_dependency 'ruby-maven', '~> 3.3.11'
|
31
|
+
s.add_development_dependency 'rspec_junit_formatter'
|
32
|
+
|
33
|
+
# Jar dependencies
|
34
|
+
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.2.1'"
|
35
|
+
s.add_runtime_dependency 'jar-dependencies'
|
36
|
+
end
|
@@ -1,121 +1,121 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require_relative "../../spec_helpers.rb"
|
3
|
-
require 'logstash/outputs/kusto'
|
4
|
-
require 'logstash/outputs/kusto/ingestor'
|
5
|
-
|
6
|
-
describe LogStash::Outputs::Kusto::Ingestor do
|
7
|
-
|
8
|
-
let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
|
9
|
-
let(:app_id) { "myid" }
|
10
|
-
let(:app_key) { LogStash::Util::Password.new("mykey") }
|
11
|
-
let(:app_tenant) { "mytenant" }
|
12
|
-
let(:database) { "mydatabase" }
|
13
|
-
let(:table) { "mytable" }
|
14
|
-
let(:proxy_host) { "localhost" }
|
15
|
-
let(:proxy_port) { 80 }
|
16
|
-
let(:proxy_protocol) { "http" }
|
17
|
-
let(:json_mapping) { "mymapping" }
|
18
|
-
let(:delete_local) { false }
|
19
|
-
let(:logger) { spy('logger') }
|
20
|
-
|
21
|
-
describe '#initialize' do
|
22
|
-
|
23
|
-
it 'does not throw an error when initializing' do
|
24
|
-
# note that this will cause an internal error since connection is being tried.
|
25
|
-
# however we still want to test that all the java stuff is working as expected
|
26
|
-
expect {
|
27
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
|
28
|
-
ingestor.stop
|
29
|
-
}.not_to raise_error
|
30
|
-
end
|
31
|
-
|
32
|
-
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
33
|
-
|
34
|
-
context 'doesnt allow database to have some dynamic part' do
|
35
|
-
dynamic_name_array.each do |test_database|
|
36
|
-
it "with database: #{test_database}" do
|
37
|
-
expect {
|
38
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
39
|
-
ingestor.stop
|
40
|
-
}.to raise_error(LogStash::ConfigurationError)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
context 'doesnt allow table to have some dynamic part' do
|
46
|
-
dynamic_name_array.each do |test_table|
|
47
|
-
it "with database: #{test_table}" do
|
48
|
-
expect {
|
49
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
50
|
-
ingestor.stop
|
51
|
-
}.to raise_error(LogStash::ConfigurationError)
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
context 'doesnt allow mapping to have some dynamic part' do
|
57
|
-
dynamic_name_array.each do |json_mapping|
|
58
|
-
it "with database: #{json_mapping}" do
|
59
|
-
expect {
|
60
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
61
|
-
ingestor.stop
|
62
|
-
}.to raise_error(LogStash::ConfigurationError)
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
context 'proxy protocol has to be http or https' do
|
68
|
-
it "with proxy protocol: socks" do
|
69
|
-
expect {
|
70
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
|
71
|
-
ingestor.stop
|
72
|
-
}.to raise_error(LogStash::ConfigurationError)
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
end
|
77
|
-
|
78
|
-
# describe 'receiving events' do
|
79
|
-
|
80
|
-
# context 'with non-zero flush interval' do
|
81
|
-
# let(:temporary_output_file) { Stud::Temporary.pathname }
|
82
|
-
|
83
|
-
# let(:event_count) { 100 }
|
84
|
-
# let(:flush_interval) { 5 }
|
85
|
-
|
86
|
-
# let(:events) do
|
87
|
-
# event_count.times.map do |idx|
|
88
|
-
# LogStash::Event.new('subject' => idx)
|
89
|
-
# end
|
90
|
-
# end
|
91
|
-
|
92
|
-
# let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
|
93
|
-
|
94
|
-
# before(:each) { output.register }
|
95
|
-
|
96
|
-
# after(:each) do
|
97
|
-
# output.close
|
98
|
-
# File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
|
99
|
-
# File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
|
100
|
-
# end
|
101
|
-
|
102
|
-
# it 'eventually flushes without receiving additional events' do
|
103
|
-
# output.multi_receive_encoded(events)
|
104
|
-
|
105
|
-
# # events should not all be flushed just yet...
|
106
|
-
# expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
|
107
|
-
# contents && contents.lines.count < event_count
|
108
|
-
# end
|
109
|
-
|
110
|
-
# # wait for the flusher to run...
|
111
|
-
# sleep(flush_interval + 1)
|
112
|
-
|
113
|
-
# # events should all be flushed
|
114
|
-
# expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
|
115
|
-
# contents && contents.lines.count == event_count
|
116
|
-
# end
|
117
|
-
# end
|
118
|
-
# end
|
119
|
-
|
120
|
-
# end
|
121
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative "../../spec_helpers.rb"
|
3
|
+
require 'logstash/outputs/kusto'
|
4
|
+
require 'logstash/outputs/kusto/ingestor'
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Kusto::Ingestor do
|
7
|
+
|
8
|
+
let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
|
9
|
+
let(:app_id) { "myid" }
|
10
|
+
let(:app_key) { LogStash::Util::Password.new("mykey") }
|
11
|
+
let(:app_tenant) { "mytenant" }
|
12
|
+
let(:database) { "mydatabase" }
|
13
|
+
let(:table) { "mytable" }
|
14
|
+
let(:proxy_host) { "localhost" }
|
15
|
+
let(:proxy_port) { 80 }
|
16
|
+
let(:proxy_protocol) { "http" }
|
17
|
+
let(:json_mapping) { "mymapping" }
|
18
|
+
let(:delete_local) { false }
|
19
|
+
let(:logger) { spy('logger') }
|
20
|
+
|
21
|
+
describe '#initialize' do
|
22
|
+
|
23
|
+
it 'does not throw an error when initializing' do
|
24
|
+
# note that this will cause an internal error since connection is being tried.
|
25
|
+
# however we still want to test that all the java stuff is working as expected
|
26
|
+
expect {
|
27
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
|
28
|
+
ingestor.stop
|
29
|
+
}.not_to raise_error
|
30
|
+
end
|
31
|
+
|
32
|
+
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
33
|
+
|
34
|
+
context 'doesnt allow database to have some dynamic part' do
|
35
|
+
dynamic_name_array.each do |test_database|
|
36
|
+
it "with database: #{test_database}" do
|
37
|
+
expect {
|
38
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
39
|
+
ingestor.stop
|
40
|
+
}.to raise_error(LogStash::ConfigurationError)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
context 'doesnt allow table to have some dynamic part' do
|
46
|
+
dynamic_name_array.each do |test_table|
|
47
|
+
it "with database: #{test_table}" do
|
48
|
+
expect {
|
49
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
50
|
+
ingestor.stop
|
51
|
+
}.to raise_error(LogStash::ConfigurationError)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
context 'doesnt allow mapping to have some dynamic part' do
|
57
|
+
dynamic_name_array.each do |json_mapping|
|
58
|
+
it "with database: #{json_mapping}" do
|
59
|
+
expect {
|
60
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
61
|
+
ingestor.stop
|
62
|
+
}.to raise_error(LogStash::ConfigurationError)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context 'proxy protocol has to be http or https' do
|
68
|
+
it "with proxy protocol: socks" do
|
69
|
+
expect {
|
70
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
|
71
|
+
ingestor.stop
|
72
|
+
}.to raise_error(LogStash::ConfigurationError)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
# describe 'receiving events' do
|
79
|
+
|
80
|
+
# context 'with non-zero flush interval' do
|
81
|
+
# let(:temporary_output_file) { Stud::Temporary.pathname }
|
82
|
+
|
83
|
+
# let(:event_count) { 100 }
|
84
|
+
# let(:flush_interval) { 5 }
|
85
|
+
|
86
|
+
# let(:events) do
|
87
|
+
# event_count.times.map do |idx|
|
88
|
+
# LogStash::Event.new('subject' => idx)
|
89
|
+
# end
|
90
|
+
# end
|
91
|
+
|
92
|
+
# let(:output) { described_class.new(options.merge( {'path' => temporary_output_file, 'flush_interval' => flush_interval, 'delete_temp_files' => false } )) }
|
93
|
+
|
94
|
+
# before(:each) { output.register }
|
95
|
+
|
96
|
+
# after(:each) do
|
97
|
+
# output.close
|
98
|
+
# File.exist?(temporary_output_file) && File.unlink(temporary_output_file)
|
99
|
+
# File.exist?(temporary_output_file + '.kusto') && File.unlink(temporary_output_file + '.kusto')
|
100
|
+
# end
|
101
|
+
|
102
|
+
# it 'eventually flushes without receiving additional events' do
|
103
|
+
# output.multi_receive_encoded(events)
|
104
|
+
|
105
|
+
# # events should not all be flushed just yet...
|
106
|
+
# expect(File.read(temporary_output_file)).to satisfy("have less than #{event_count} lines") do |contents|
|
107
|
+
# contents && contents.lines.count < event_count
|
108
|
+
# end
|
109
|
+
|
110
|
+
# # wait for the flusher to run...
|
111
|
+
# sleep(flush_interval + 1)
|
112
|
+
|
113
|
+
# # events should all be flushed
|
114
|
+
# expect(File.read(temporary_output_file)).to satisfy("have exactly #{event_count} lines") do |contents|
|
115
|
+
# contents && contents.lines.count == event_count
|
116
|
+
# end
|
117
|
+
# end
|
118
|
+
# end
|
119
|
+
|
120
|
+
# end
|
121
|
+
end
|
data/spec/outputs/kusto_spec.rb
CHANGED
@@ -1,56 +1,56 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require 'logstash/outputs/kusto'
|
3
|
-
require 'logstash/codecs/plain'
|
4
|
-
require 'logstash/event'
|
5
|
-
|
6
|
-
describe LogStash::Outputs::Kusto do
|
7
|
-
|
8
|
-
let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
|
9
|
-
"ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
|
10
|
-
"app_id" => "myid",
|
11
|
-
"app_key" => "mykey",
|
12
|
-
"app_tenant" => "mytenant",
|
13
|
-
"database" => "mydatabase",
|
14
|
-
"table" => "mytable",
|
15
|
-
"json_mapping" => "mymapping",
|
16
|
-
"proxy_host" => "localhost",
|
17
|
-
"proxy_port" => 3128,
|
18
|
-
"proxy_protocol" => "https"
|
19
|
-
} }
|
20
|
-
|
21
|
-
describe '#register' do
|
22
|
-
|
23
|
-
it 'doesnt allow the path to start with a dynamic string' do
|
24
|
-
kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
|
25
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
26
|
-
kusto.close
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'path must include a dynamic string to allow file rotation' do
|
30
|
-
kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
|
31
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
32
|
-
kusto.close
|
33
|
-
end
|
34
|
-
|
35
|
-
|
36
|
-
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
37
|
-
|
38
|
-
context 'doesnt allow the root directory to have some dynamic part' do
|
39
|
-
dynamic_name_array.each do |test_path|
|
40
|
-
it "with path: #{test_path}" do
|
41
|
-
kusto = described_class.new(options.merge( {'path' => test_path} ))
|
42
|
-
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
43
|
-
kusto.close
|
44
|
-
end
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
|
-
it 'allow to have dynamic part after the file root' do
|
49
|
-
kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
|
50
|
-
expect { kusto.register }.not_to raise_error
|
51
|
-
kusto.close
|
52
|
-
end
|
53
|
-
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'logstash/outputs/kusto'
|
3
|
+
require 'logstash/codecs/plain'
|
4
|
+
require 'logstash/event'
|
5
|
+
|
6
|
+
describe LogStash::Outputs::Kusto do
|
7
|
+
|
8
|
+
let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
|
9
|
+
"ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
|
10
|
+
"app_id" => "myid",
|
11
|
+
"app_key" => "mykey",
|
12
|
+
"app_tenant" => "mytenant",
|
13
|
+
"database" => "mydatabase",
|
14
|
+
"table" => "mytable",
|
15
|
+
"json_mapping" => "mymapping",
|
16
|
+
"proxy_host" => "localhost",
|
17
|
+
"proxy_port" => 3128,
|
18
|
+
"proxy_protocol" => "https"
|
19
|
+
} }
|
20
|
+
|
21
|
+
describe '#register' do
|
22
|
+
|
23
|
+
it 'doesnt allow the path to start with a dynamic string' do
|
24
|
+
kusto = described_class.new(options.merge( {'path' => '/%{name}'} ))
|
25
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
26
|
+
kusto.close
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'path must include a dynamic string to allow file rotation' do
|
30
|
+
kusto = described_class.new(options.merge( {'path' => '/{name}'} ))
|
31
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
32
|
+
kusto.close
|
33
|
+
end
|
34
|
+
|
35
|
+
|
36
|
+
dynamic_name_array = ['/a%{name}/', '/a %{name}/', '/a- %{name}/', '/a- %{name}']
|
37
|
+
|
38
|
+
context 'doesnt allow the root directory to have some dynamic part' do
|
39
|
+
dynamic_name_array.each do |test_path|
|
40
|
+
it "with path: #{test_path}" do
|
41
|
+
kusto = described_class.new(options.merge( {'path' => test_path} ))
|
42
|
+
expect { kusto.register }.to raise_error(LogStash::ConfigurationError)
|
43
|
+
kusto.close
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'allow to have dynamic part after the file root' do
|
49
|
+
kusto = described_class.new(options.merge({'path' => '/tmp/%{name}'}))
|
50
|
+
expect { kusto.register }.not_to raise_error
|
51
|
+
kusto.close
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
data/spec/spec_helpers.rb
CHANGED
@@ -1,21 +1,21 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
require "logstash/devutils/rspec/spec_helper"
|
3
|
-
require "logstash/logging/logger"
|
4
|
-
|
5
|
-
LogStash::Logging::Logger::configure_logging("debug")
|
6
|
-
|
7
|
-
RSpec.configure do |config|
|
8
|
-
# register around filter that captures stdout and stderr
|
9
|
-
config.around(:each) do |example|
|
10
|
-
$stdout = StringIO.new
|
11
|
-
$stderr = StringIO.new
|
12
|
-
|
13
|
-
example.run
|
14
|
-
|
15
|
-
example.metadata[:stdout] = $stdout.string
|
16
|
-
example.metadata[:stderr] = $stderr.string
|
17
|
-
|
18
|
-
$stdout = STDOUT
|
19
|
-
$stderr = STDERR
|
20
|
-
end
|
21
|
-
end
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/logging/logger"
|
4
|
+
|
5
|
+
LogStash::Logging::Logger::configure_logging("debug")
|
6
|
+
|
7
|
+
RSpec.configure do |config|
|
8
|
+
# register around filter that captures stdout and stderr
|
9
|
+
config.around(:each) do |example|
|
10
|
+
$stdout = StringIO.new
|
11
|
+
$stderr = StringIO.new
|
12
|
+
|
13
|
+
example.run
|
14
|
+
|
15
|
+
example.metadata[:stdout] = $stdout.string
|
16
|
+
example.metadata[:stderr] = $stderr.string
|
17
|
+
|
18
|
+
$stdout = STDOUT
|
19
|
+
$stderr = STDERR
|
20
|
+
end
|
21
|
+
end
|