logstash-output-kusto 1.0.2-java → 1.0.5-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/Gemfile +2 -0
- data/README.md +15 -0
- data/SECURITY.md +41 -0
- data/lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar +0 -0
- data/lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar +0 -0
- data/lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar +0 -0
- data/lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar +0 -0
- data/lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar +0 -0
- data/lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar +0 -0
- data/lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar +0 -0
- data/lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar +0 -0
- data/lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar +0 -0
- data/lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar +0 -0
- data/lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar +0 -0
- data/lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar +0 -0
- data/lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar +0 -0
- data/lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar +0 -0
- data/lib/com/nimbusds/content-type/2.1/content-type-2.1.jar +0 -0
- data/lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar +0 -0
- data/lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar +0 -0
- data/lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar +0 -0
- data/lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar +0 -0
- data/lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar +0 -0
- data/lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar +0 -0
- data/lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar +0 -0
- data/lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar +0 -0
- data/lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar +0 -0
- data/lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar +0 -0
- data/lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar +0 -0
- data/lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar +0 -0
- data/lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar +0 -0
- data/lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar +0 -0
- data/lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar +0 -0
- data/lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar +0 -0
- data/lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar +0 -0
- data/lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar +0 -0
- data/lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar +0 -0
- data/lib/logstash/outputs/kusto/ingestor.rb +32 -17
- data/lib/logstash/outputs/kusto.rb +11 -2
- data/lib/logstash-output-kusto_jars.rb +136 -50
- data/lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar +0 -0
- data/lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar +0 -0
- data/lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar +0 -0
- data/lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar +0 -0
- data/lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar +0 -0
- data/lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar +0 -0
- data/lib/org/apache/httpcomponents/httpclient/{4.5.8/httpclient-4.5.8.jar → 4.5.13/httpclient-4.5.13.jar} +0 -0
- data/lib/org/apache/httpcomponents/httpcore/{4.4.11/httpcore-4.4.11.jar → 4.4.15/httpcore-4.4.15.jar} +0 -0
- data/lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar +0 -0
- data/lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar +0 -0
- data/lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar +0 -0
- data/lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar +0 -0
- data/lib/org/json/json/20201115/json-20201115.jar +0 -0
- data/lib/org/ow2/asm/asm/9.1/asm-9.1.jar +0 -0
- data/lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar +0 -0
- data/lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar +0 -0
- data/logstash-output-kusto.gemspec +3 -2
- data/spec/outputs/kusto/ingestor_spec.rb +19 -7
- data/spec/outputs/kusto_spec.rb +7 -5
- data/spec/spec_helpers.rb +1 -1
- metadata +94 -36
- data/lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar +0 -0
- data/lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar +0 -0
- data/lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar +0 -0
- data/lib/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
- data/lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar +0 -0
- data/lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar +0 -0
- data/lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar +0 -0
- data/lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar +0 -0
- data/lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar +0 -0
- data/lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar +0 -0
- data/lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar +0 -0
- data/lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar +0 -0
- data/lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar +0 -0
- data/lib/javax/activation/activation/1.1/activation-1.1.jar +0 -0
- data/lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar +0 -0
- data/lib/net/minidev/json-smart/2.3/json-smart-2.3.jar +0 -0
- data/lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar +0 -0
- data/lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar +0 -0
- data/lib/org/json/json/20190722/json-20190722.jar +0 -0
- data/lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 40d2d10177bc99b9da270d95a843ca5490689ac6a9bbdd548744d041e29a557c
|
4
|
+
data.tar.gz: b5f85499389d432eb0e60e5be23bb75ea7b9c5c43df6a1d1a7e07f3b5fd44c57
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82e08571c9c68c67dd7e2c8f653cfce159b01902cee31982ddefa9cea59769510acb25ebf5d3ae2ed18354a7ccae1ba0dcc6b6135940a706447b017d80d520fb
|
7
|
+
data.tar.gz: b3c7c5860bfe0fea8e7da8d5e5356a56b6dd54733214f46c6492e452580c635da637be223a80723db05c2c496edda4d8b607ebe47138a582b0b1fc8be7e5bd4a
|
data/CHANGELOG.md
CHANGED
@@ -28,4 +28,10 @@
|
|
28
28
|
# 1.0.0
|
29
29
|
|
30
30
|
- Use stable (2.1.2) version of the java sdk, and retrieve it from maven with bundler.
|
31
|
-
- Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
|
31
|
+
- Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
|
32
|
+
|
33
|
+
|
34
|
+
# 1.0.5
|
35
|
+
|
36
|
+
- Use (3.1.3) version of the java sdk, and retrieve it from maven with bundler.
|
37
|
+
- Added support for `proxy_host` `proxy_port` `proxy_protocol` to support proxying ingestion to Kusto
|
data/Gemfile
CHANGED
@@ -14,6 +14,8 @@ if Dir.exist?(logstash_path) && use_logstash_source
|
|
14
14
|
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
15
15
|
else
|
16
16
|
puts "using default logstash"
|
17
|
+
gem 'logstash-core'
|
18
|
+
gem 'logstash-core-plugin-api'
|
17
19
|
end
|
18
20
|
|
19
21
|
gem "rspec"
|
data/README.md
CHANGED
@@ -39,6 +39,9 @@ output {
|
|
39
39
|
database => "<database name>"
|
40
40
|
table => "<target table>"
|
41
41
|
json_mapping => "<mapping name>"
|
42
|
+
proxy_host => "<proxy host>"
|
43
|
+
proxy_port => <proxy port>
|
44
|
+
proxy_protocol => <"http"|"https">
|
42
45
|
}
|
43
46
|
}
|
44
47
|
```
|
@@ -57,6 +60,16 @@ More information about configuring Logstash can be found in the [logstash config
|
|
57
60
|
| **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files found in the path upon startup | |
|
58
61
|
| **delete_temp_files** | Determines if temp files will be deleted after a successful upload (true is default; set false for debug purposes only)| |
|
59
62
|
| **flush_interval** | The time (in seconds) for flushing writes to temporary files. Default is 2 seconds, 0 will flush on every event. Increase this value to reduce IO calls but keep in mind that events in the buffer will be lost in case of abrupt failure.| |
|
63
|
+
| **proxy_host** | The proxy hostname for redirecting traffic to Kusto.| |
|
64
|
+
| **proxy_port** | The proxy port for the proxy. Defaults to 80.| |
|
65
|
+
| **proxy_protocol** | The proxy server protocol , is one of http or https.| |
|
66
|
+
|
67
|
+
> Note : LS_JAVA_OPTS can be used to set proxy parameters as well (using export or SET options)
|
68
|
+
|
69
|
+
```bash
|
70
|
+
export LS_JAVA_OPTS="-Dhttp.proxyHost=1.2.34 -Dhttp.proxyPort=8989 -Dhttps.proxyHost=1.2.3.4 -Dhttps.proxyPort=8989"
|
71
|
+
```
|
72
|
+
|
60
73
|
|
61
74
|
## Development Requirements
|
62
75
|
|
@@ -64,6 +77,8 @@ More information about configuring Logstash can be found in the [logstash config
|
|
64
77
|
- JRuby 9.2 or higher, defined with openjdk 8 64bit
|
65
78
|
- Logstash, defined with openjdk 8 64bit
|
66
79
|
|
80
|
+
*It is reccomened to use the bundled jdk and jruby with logstash to avoid compatibility issues.*
|
81
|
+
|
67
82
|
To fully build the gem, run:
|
68
83
|
|
69
84
|
```shell
|
data/SECURITY.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
|
2
|
+
|
3
|
+
## Security
|
4
|
+
|
5
|
+
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
|
6
|
+
|
7
|
+
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
|
8
|
+
|
9
|
+
## Reporting Security Issues
|
10
|
+
|
11
|
+
**Please do not report security vulnerabilities through public GitHub issues.**
|
12
|
+
|
13
|
+
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
|
14
|
+
|
15
|
+
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
|
16
|
+
|
17
|
+
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
|
18
|
+
|
19
|
+
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
|
20
|
+
|
21
|
+
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
|
22
|
+
* Full paths of source file(s) related to the manifestation of the issue
|
23
|
+
* The location of the affected source code (tag/branch/commit or direct URL)
|
24
|
+
* Any special configuration required to reproduce the issue
|
25
|
+
* Step-by-step instructions to reproduce the issue
|
26
|
+
* Proof-of-concept or exploit code (if possible)
|
27
|
+
* Impact of the issue, including how an attacker might exploit the issue
|
28
|
+
|
29
|
+
This information will help us triage your report more quickly.
|
30
|
+
|
31
|
+
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
|
32
|
+
|
33
|
+
## Preferred Languages
|
34
|
+
|
35
|
+
We prefer all communications to be in English.
|
36
|
+
|
37
|
+
## Policy
|
38
|
+
|
39
|
+
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
|
40
|
+
|
41
|
+
<!-- END MICROSOFT SECURITY.MD BLOCK -->
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'logstash/outputs/base'
|
3
|
+
require 'logstash/outputs/base'
|
4
4
|
require 'logstash/namespace'
|
5
5
|
require 'logstash/errors'
|
6
6
|
|
@@ -20,33 +20,40 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
20
20
|
LOW_QUEUE_LENGTH = 3
|
21
21
|
FIELD_REF = /%\{[^}]+\}/
|
22
22
|
|
23
|
-
def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger, threadpool = DEFAULT_THREADPOOL)
|
23
|
+
def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
|
24
24
|
@workers_pool = threadpool
|
25
25
|
@logger = logger
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
@logger.debug('Preparing Kusto resources.')
|
26
|
+
validate_config(database, table, json_mapping,proxy_protocol)
|
27
|
+
@logger.info('Preparing Kusto resources.')
|
30
28
|
|
31
29
|
kusto_java = Java::com.microsoft.azure.kusto
|
32
|
-
|
30
|
+
apache_http = Java::org.apache.http
|
31
|
+
kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
|
32
|
+
#
|
33
33
|
@logger.debug(Gem.loaded_specs.to_s)
|
34
34
|
# Unfortunately there's no way to avoid using the gem/plugin name directly...
|
35
35
|
name_for_tracing = "logstash-output-kusto:#{Gem.loaded_specs['logstash-output-kusto']&.version || "unknown"}"
|
36
36
|
@logger.debug("Client name for tracing: #{name_for_tracing}")
|
37
|
-
kusto_connection_string.setClientVersionForTracing(name_for_tracing)
|
38
|
-
|
39
|
-
@kusto_client =
|
37
|
+
kusto_connection_string.setClientVersionForTracing(name_for_tracing)
|
38
|
+
|
39
|
+
@kusto_client = begin
|
40
|
+
if proxy_host.nil? || proxy_host.empty?
|
41
|
+
kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
|
42
|
+
else
|
43
|
+
kusto_http_client_properties = kusto_java.data.HttpClientProperties.builder().proxy(apache_http.HttpHost.new(proxy_host,proxy_port,proxy_protocol)).build()
|
44
|
+
kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string, kusto_http_client_properties)
|
45
|
+
end
|
46
|
+
end
|
40
47
|
|
41
48
|
@ingestion_properties = kusto_java.ingest.IngestionProperties.new(database, table)
|
42
|
-
@ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::
|
43
|
-
@ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::
|
49
|
+
@ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON)
|
50
|
+
@ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON)
|
44
51
|
@delete_local = delete_local
|
45
52
|
|
46
53
|
@logger.debug('Kusto resources are ready.')
|
47
54
|
end
|
48
55
|
|
49
|
-
def validate_config(database, table, json_mapping)
|
56
|
+
def validate_config(database, table, json_mapping,proxy_protocol)
|
50
57
|
if database =~ FIELD_REF
|
51
58
|
@logger.error('database config value should not be dynamic.', database)
|
52
59
|
raise LogStash::ConfigurationError.new('database config value should not be dynamic.')
|
@@ -61,6 +68,12 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
61
68
|
@logger.error('json_mapping config value should not be dynamic.', json_mapping)
|
62
69
|
raise LogStash::ConfigurationError.new('json_mapping config value should not be dynamic.')
|
63
70
|
end
|
71
|
+
|
72
|
+
if not(["https", "http"].include? proxy_protocol)
|
73
|
+
@logger.error('proxy_protocol has to be http or https.', proxy_protocol)
|
74
|
+
raise LogStash::ConfigurationError.new('proxy_protocol has to be http or https.')
|
75
|
+
end
|
76
|
+
|
64
77
|
end
|
65
78
|
|
66
79
|
def upload_async(path, delete_on_success)
|
@@ -95,11 +108,13 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
95
108
|
# local_ingestion_properties.addJsonMappingName(json_mapping)
|
96
109
|
# end
|
97
110
|
|
98
|
-
|
99
|
-
|
100
|
-
|
111
|
+
if file_size > 0
|
112
|
+
file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
|
113
|
+
@kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
|
114
|
+
else
|
115
|
+
@logger.warn("File #{path} is an empty file and is not ingested.")
|
116
|
+
end
|
101
117
|
File.delete(path) if delete_on_success
|
102
|
-
|
103
118
|
@logger.debug("File #{path} sent to kusto.")
|
104
119
|
rescue Errno::ENOENT => e
|
105
120
|
@logger.error("File doesn't exist! Unrecoverable error.", exception: e.class, message: e.message, path: path, backtrace: e.backtrace)
|
@@ -88,7 +88,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
88
88
|
# Note that this must be in JSON format, as this is the interface between Logstash and Kusto
|
89
89
|
config :json_mapping, validate: :string, required: true
|
90
90
|
|
91
|
-
#
|
91
|
+
# Mapping name - deprecated, use json_mapping
|
92
92
|
config :mapping, validate: :string, deprecated: true
|
93
93
|
|
94
94
|
|
@@ -106,6 +106,15 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
106
106
|
# starts processing them in the main thread (not healthy)
|
107
107
|
config :upload_queue_size, validate: :number, default: 30
|
108
108
|
|
109
|
+
# Host of the proxy , is an optional field. Can connect directly
|
110
|
+
config :proxy_host, validate: :string, required: false
|
111
|
+
|
112
|
+
# Port where the proxy runs , defaults to 80. Usually a value like 3128
|
113
|
+
config :proxy_port, validate: :number, required: false , default: 80
|
114
|
+
|
115
|
+
# Check Proxy URL can be over http or https. Dowe need it this way or ignore this & remove this
|
116
|
+
config :proxy_protocol, validate: :string, required: false , default: 'http'
|
117
|
+
|
109
118
|
default :codec, 'json_lines'
|
110
119
|
|
111
120
|
def register
|
@@ -141,7 +150,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
|
|
141
150
|
max_queue: upload_queue_size,
|
142
151
|
fallback_policy: :caller_runs)
|
143
152
|
|
144
|
-
@ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, @logger, executor)
|
153
|
+
@ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor)
|
145
154
|
|
146
155
|
# send existing files
|
147
156
|
recover_past_files if recovery
|
@@ -2,63 +2,149 @@
|
|
2
2
|
begin
|
3
3
|
require 'jar_dependencies'
|
4
4
|
rescue LoadError
|
5
|
+
require 'io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar'
|
6
|
+
require 'com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar'
|
7
|
+
require 'com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar'
|
8
|
+
require 'com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar'
|
9
|
+
require 'com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar'
|
10
|
+
require 'io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar'
|
11
|
+
require 'net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar'
|
12
|
+
require 'io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar'
|
13
|
+
require 'org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar'
|
14
|
+
require 'io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar'
|
15
|
+
require 'org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar'
|
16
|
+
require 'org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar'
|
17
|
+
require 'com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar'
|
18
|
+
require 'io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar'
|
19
|
+
require 'io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar'
|
20
|
+
require 'com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar'
|
21
|
+
require 'io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar'
|
22
|
+
require 'org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar'
|
23
|
+
require 'io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar'
|
24
|
+
require 'io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar'
|
25
|
+
require 'io/vavr/vavr/0.10.2/vavr-0.10.2.jar'
|
26
|
+
require 'io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar'
|
27
|
+
require 'com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar'
|
28
|
+
require 'com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar'
|
29
|
+
require 'org/apache/commons/commons-text/1.9/commons-text-1.9.jar'
|
30
|
+
require 'net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar'
|
31
|
+
require 'com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar'
|
32
|
+
require 'io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar'
|
33
|
+
require 'jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar'
|
34
|
+
require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
|
35
|
+
require 'net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar'
|
36
|
+
require 'io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar'
|
37
|
+
require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
|
38
|
+
require 'com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar'
|
39
|
+
require 'com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar'
|
5
40
|
require 'com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar'
|
6
|
-
require 'com/
|
7
|
-
require '
|
8
|
-
require '
|
9
|
-
require '
|
10
|
-
require 'com/
|
11
|
-
require '
|
12
|
-
require '
|
13
|
-
require '
|
14
|
-
require '
|
15
|
-
require 'com/
|
16
|
-
require '
|
17
|
-
require '
|
18
|
-
require 'com/
|
19
|
-
require '
|
20
|
-
require 'org/
|
21
|
-
require '
|
22
|
-
require '
|
41
|
+
require 'com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar'
|
42
|
+
require 'io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar'
|
43
|
+
require 'io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar'
|
44
|
+
require 'com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar'
|
45
|
+
require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
|
46
|
+
require 'org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar'
|
47
|
+
require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
|
48
|
+
require 'com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar'
|
49
|
+
require 'io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar'
|
50
|
+
require 'com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar'
|
51
|
+
require 'org/ow2/asm/asm/9.1/asm-9.1.jar'
|
52
|
+
require 'com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar'
|
53
|
+
require 'com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar'
|
54
|
+
require 'org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar'
|
55
|
+
require 'org/json/json/20201115/json-20201115.jar'
|
56
|
+
require 'io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar'
|
57
|
+
require 'io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar'
|
58
|
+
require 'net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar'
|
59
|
+
require 'io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar'
|
60
|
+
require 'io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar'
|
61
|
+
require 'jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar'
|
62
|
+
require 'io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar'
|
63
|
+
require 'com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar'
|
64
|
+
require 'com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar'
|
65
|
+
require 'io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar'
|
66
|
+
require 'com/nimbusds/content-type/2.1/content-type-2.1.jar'
|
23
67
|
require 'org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar'
|
24
|
-
require '
|
68
|
+
require 'com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar'
|
25
69
|
require 'com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar'
|
26
|
-
require 'com/
|
27
|
-
require 'com/
|
28
|
-
require '
|
29
|
-
require '
|
30
|
-
require '
|
31
|
-
require '
|
32
|
-
require 'org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar'
|
70
|
+
require 'com/azure/azure-core/1.21.0/azure-core-1.21.0.jar'
|
71
|
+
require 'com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar'
|
72
|
+
require 'org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar'
|
73
|
+
require 'io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar'
|
74
|
+
require 'org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar'
|
75
|
+
require 'io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar'
|
33
76
|
end
|
34
77
|
|
35
78
|
if defined? Jars
|
79
|
+
require_jar 'io.netty', 'netty-resolver-dns-native-macos', 'osx-x86_64', '4.1.68.Final'
|
80
|
+
require_jar 'com.google.guava', 'guava', '24.1.1-jre'
|
81
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.1.3'
|
82
|
+
require_jar 'com.microsoft.azure', 'azure-storage', '8.6.6'
|
83
|
+
require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.1.3'
|
84
|
+
require_jar 'io.projectreactor.netty', 'reactor-netty-http', '1.0.11'
|
85
|
+
require_jar 'net.java.dev.jna', 'jna', '5.5.0'
|
86
|
+
require_jar 'io.netty', 'netty-handler-proxy', '4.1.68.Final'
|
87
|
+
require_jar 'org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'
|
88
|
+
require_jar 'io.netty', 'netty-codec-socks', '4.1.68.Final'
|
89
|
+
require_jar 'org.apache.commons', 'commons-lang3', '3.11'
|
90
|
+
require_jar 'org.reactivestreams', 'reactive-streams', '1.0.3'
|
91
|
+
require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.2.4'
|
92
|
+
require_jar 'io.github.resilience4j', 'resilience4j-core', '1.7.1'
|
93
|
+
require_jar 'io.netty', 'netty-codec-dns', '4.1.68.Final'
|
94
|
+
require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.9.3'
|
95
|
+
require_jar 'io.netty', 'netty-transport', '4.1.68.Final'
|
96
|
+
require_jar 'org.jetbrains', 'annotations', '22.0.0'
|
97
|
+
require_jar 'io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'
|
98
|
+
require_jar 'io.netty', 'netty-transport-native-epoll', 'linux-x86_64', '4.1.68.Final'
|
99
|
+
require_jar 'io.vavr', 'vavr', '0.10.2'
|
100
|
+
require_jar 'io.netty', 'netty-resolver-dns', '4.1.68.Final'
|
101
|
+
require_jar 'com.azure', 'azure-identity', '1.3.7'
|
102
|
+
require_jar 'com.fasterxml.jackson.module', 'jackson-module-jaxb-annotations', '2.12.5'
|
103
|
+
require_jar 'org.apache.commons', 'commons-text', '1.9'
|
104
|
+
require_jar 'net.java.dev.jna', 'jna-platform', '5.6.0'
|
105
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.13.0'
|
106
|
+
require_jar 'io.netty', 'netty-resolver', '4.1.68.Final'
|
107
|
+
require_jar 'jakarta.activation', 'jakarta.activation-api', '1.2.1'
|
108
|
+
require_jar 'commons-logging', 'commons-logging', '1.2'
|
109
|
+
require_jar 'net.minidev', 'accessors-smart', '2.4.7'
|
110
|
+
require_jar 'io.netty', 'netty-codec', '4.1.68.Final'
|
111
|
+
require_jar 'commons-codec', 'commons-codec', '1.11'
|
112
|
+
require_jar 'com.microsoft.azure', 'msal4j-persistence-extension', '1.1.0'
|
113
|
+
require_jar 'com.fasterxml.woodstox', 'woodstox-core', '6.2.4'
|
36
114
|
require_jar 'com.github.stephenc.jcip', 'jcip-annotations', '1.0-1'
|
37
|
-
require_jar 'com.
|
38
|
-
require_jar '
|
39
|
-
require_jar '
|
40
|
-
require_jar '
|
41
|
-
require_jar 'com.
|
42
|
-
require_jar '
|
43
|
-
require_jar '
|
44
|
-
require_jar '
|
45
|
-
require_jar '
|
46
|
-
require_jar 'com.
|
47
|
-
require_jar '
|
48
|
-
require_jar '
|
49
|
-
require_jar 'com.
|
50
|
-
require_jar '
|
51
|
-
require_jar 'org.
|
52
|
-
require_jar '
|
53
|
-
require_jar '
|
115
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.13.0'
|
116
|
+
require_jar 'io.netty', 'netty-transport-native-kqueue', 'osx-x86_64', '4.1.68.Final'
|
117
|
+
require_jar 'io.netty', 'netty-handler', '4.1.68.Final'
|
118
|
+
require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.13.0'
|
119
|
+
require_jar 'com.google.code.findbugs', 'jsr305', '1.3.9'
|
120
|
+
require_jar 'org.checkerframework', 'checker-compat-qual', '2.0.0'
|
121
|
+
require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
|
122
|
+
require_jar 'com.google.j2objc', 'j2objc-annotations', '1.1'
|
123
|
+
require_jar 'io.netty', 'netty-codec-http', '4.1.68.Final'
|
124
|
+
require_jar 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-xml', '2.12.5'
|
125
|
+
require_jar 'org.ow2.asm', 'asm', '9.1'
|
126
|
+
require_jar 'com.azure', 'azure-core-http-netty', '1.11.1'
|
127
|
+
require_jar 'com.fasterxml.jackson.datatype', 'jackson-datatype-jsr310', '2.12.5'
|
128
|
+
require_jar 'org.codehaus.woodstox', 'stax2-api', '4.2.1'
|
129
|
+
require_jar 'org.json', 'json', '20201115'
|
130
|
+
require_jar 'io.netty', 'netty-tcnative-boringssl-static', '2.0.43.Final'
|
131
|
+
require_jar 'io.netty', 'netty-common', '4.1.68.Final'
|
132
|
+
require_jar 'net.minidev', 'json-smart', '2.4.7'
|
133
|
+
require_jar 'io.projectreactor', 'reactor-core', '3.4.10'
|
134
|
+
require_jar 'io.netty', 'netty-buffer', '4.1.68.Final'
|
135
|
+
require_jar 'jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.2'
|
136
|
+
require_jar 'io.github.resilience4j', 'resilience4j-retry', '1.7.1'
|
137
|
+
require_jar 'com.google.errorprone', 'error_prone_annotations', '2.1.3'
|
138
|
+
require_jar 'com.univocity', 'univocity-parsers', '2.9.1'
|
139
|
+
require_jar 'io.vavr', 'vavr-match', '0.10.2'
|
140
|
+
require_jar 'com.nimbusds', 'content-type', '2.1'
|
54
141
|
require_jar 'org.slf4j', 'slf4j-api', '1.8.0-beta4'
|
55
|
-
require_jar '
|
142
|
+
require_jar 'com.microsoft.azure', 'msal4j', '1.11.0'
|
56
143
|
require_jar 'com.nimbusds', 'lang-tag', '1.5'
|
57
|
-
require_jar 'com.
|
58
|
-
require_jar 'com.
|
59
|
-
require_jar '
|
60
|
-
require_jar '
|
61
|
-
require_jar '
|
62
|
-
require_jar '
|
63
|
-
require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.8'
|
144
|
+
require_jar 'com.azure', 'azure-core', '1.21.0'
|
145
|
+
require_jar 'com.nimbusds', 'oauth2-oidc-sdk', '9.7'
|
146
|
+
require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.15'
|
147
|
+
require_jar 'io.netty', 'netty-codec-http2', '4.1.68.Final'
|
148
|
+
require_jar 'org.slf4j', 'slf4j-simple', '1.8.0-beta4'
|
149
|
+
require_jar 'io.projectreactor.netty', 'reactor-netty-core', '1.0.11'
|
64
150
|
end
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
|
3
|
-
s.version = '1.0.
|
3
|
+
s.version = '1.0.5'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Writes events to Azure Data Explorer (Kusto)'
|
6
6
|
s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
|
@@ -28,8 +28,9 @@ Gem::Specification.new do |s|
|
|
28
28
|
s.add_development_dependency 'flores'
|
29
29
|
s.add_development_dependency 'logstash-input-generator'
|
30
30
|
s.add_development_dependency 'ruby-maven', '~> 3.3.11'
|
31
|
+
s.add_development_dependency 'rspec_junit_formatter'
|
31
32
|
|
32
33
|
# Jar dependencies
|
33
|
-
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest,
|
34
|
+
s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'"
|
34
35
|
s.add_runtime_dependency 'jar-dependencies'
|
35
36
|
end
|
@@ -5,12 +5,15 @@ require 'logstash/outputs/kusto/ingestor'
|
|
5
5
|
|
6
6
|
describe LogStash::Outputs::Kusto::Ingestor do
|
7
7
|
|
8
|
-
let(:ingest_url) { "
|
8
|
+
let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
|
9
9
|
let(:app_id) { "myid" }
|
10
10
|
let(:app_key) { LogStash::Util::Password.new("mykey") }
|
11
11
|
let(:app_tenant) { "mytenant" }
|
12
12
|
let(:database) { "mydatabase" }
|
13
13
|
let(:table) { "mytable" }
|
14
|
+
let(:proxy_host) { "localhost" }
|
15
|
+
let(:proxy_port) { 80 }
|
16
|
+
let(:proxy_protocol) { "http" }
|
14
17
|
let(:json_mapping) { "mymapping" }
|
15
18
|
let(:delete_local) { false }
|
16
19
|
let(:logger) { spy('logger') }
|
@@ -21,7 +24,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
|
|
21
24
|
# note that this will cause an internal error since connection is being tried.
|
22
25
|
# however we still want to test that all the java stuff is working as expected
|
23
26
|
expect {
|
24
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger)
|
27
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
|
25
28
|
ingestor.stop
|
26
29
|
}.not_to raise_error
|
27
30
|
end
|
@@ -32,7 +35,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
|
|
32
35
|
dynamic_name_array.each do |test_database|
|
33
36
|
it "with database: #{test_database}" do
|
34
37
|
expect {
|
35
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, logger)
|
38
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
36
39
|
ingestor.stop
|
37
40
|
}.to raise_error(LogStash::ConfigurationError)
|
38
41
|
end
|
@@ -43,7 +46,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
|
|
43
46
|
dynamic_name_array.each do |test_table|
|
44
47
|
it "with database: #{test_table}" do
|
45
48
|
expect {
|
46
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, logger)
|
49
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
47
50
|
ingestor.stop
|
48
51
|
}.to raise_error(LogStash::ConfigurationError)
|
49
52
|
end
|
@@ -51,16 +54,25 @@ describe LogStash::Outputs::Kusto::Ingestor do
|
|
51
54
|
end
|
52
55
|
|
53
56
|
context 'doesnt allow mapping to have some dynamic part' do
|
54
|
-
dynamic_name_array.each do |
|
55
|
-
it "with database: #{
|
57
|
+
dynamic_name_array.each do |json_mapping|
|
58
|
+
it "with database: #{json_mapping}" do
|
56
59
|
expect {
|
57
|
-
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table,
|
60
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
|
58
61
|
ingestor.stop
|
59
62
|
}.to raise_error(LogStash::ConfigurationError)
|
60
63
|
end
|
61
64
|
end
|
62
65
|
end
|
63
66
|
|
67
|
+
context 'proxy protocol has to be http or https' do
|
68
|
+
it "with proxy protocol: socks" do
|
69
|
+
expect {
|
70
|
+
ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
|
71
|
+
ingestor.stop
|
72
|
+
}.to raise_error(LogStash::ConfigurationError)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
64
76
|
end
|
65
77
|
|
66
78
|
# describe 'receiving events' do
|
data/spec/outputs/kusto_spec.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require 'logstash/devutils/rspec/spec_helper'
|
3
2
|
require 'logstash/outputs/kusto'
|
4
3
|
require 'logstash/codecs/plain'
|
5
4
|
require 'logstash/event'
|
@@ -7,13 +6,16 @@ require 'logstash/event'
|
|
7
6
|
describe LogStash::Outputs::Kusto do
|
8
7
|
|
9
8
|
let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
|
10
|
-
"ingest_url" => "
|
9
|
+
"ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
|
11
10
|
"app_id" => "myid",
|
12
11
|
"app_key" => "mykey",
|
13
|
-
"app_tenant" => "mytenant",
|
12
|
+
"app_tenant" => "mytenant",
|
14
13
|
"database" => "mydatabase",
|
15
14
|
"table" => "mytable",
|
16
|
-
"json_mapping" => "mymapping"
|
15
|
+
"json_mapping" => "mymapping",
|
16
|
+
"proxy_host" => "localhost",
|
17
|
+
"proxy_port" => 3128,
|
18
|
+
"proxy_protocol" => "https"
|
17
19
|
} }
|
18
20
|
|
19
21
|
describe '#register' do
|
@@ -50,5 +52,5 @@ describe LogStash::Outputs::Kusto do
|
|
50
52
|
end
|
51
53
|
|
52
54
|
end
|
53
|
-
|
55
|
+
|
54
56
|
end
|
data/spec/spec_helpers.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-kusto
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.5
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Tamir Kamara
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-08-04 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -18,8 +18,8 @@ dependencies:
|
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '2.0'
|
20
20
|
name: logstash-core-plugin-api
|
21
|
-
type: :runtime
|
22
21
|
prerelease: false
|
22
|
+
type: :runtime
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
@@ -32,8 +32,8 @@ dependencies:
|
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
name: logstash-codec-json_lines
|
35
|
-
type: :runtime
|
36
35
|
prerelease: false
|
36
|
+
type: :runtime
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - ">="
|
@@ -46,8 +46,8 @@ dependencies:
|
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
name: logstash-codec-line
|
49
|
-
type: :runtime
|
50
49
|
prerelease: false
|
50
|
+
type: :runtime
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - ">="
|
@@ -60,8 +60,8 @@ dependencies:
|
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
name: logstash-devutils
|
63
|
-
type: :development
|
64
63
|
prerelease: false
|
64
|
+
type: :development
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
67
|
- - ">="
|
@@ -74,8 +74,8 @@ dependencies:
|
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
name: flores
|
77
|
-
type: :development
|
78
77
|
prerelease: false
|
78
|
+
type: :development
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
81
|
- - ">="
|
@@ -88,8 +88,8 @@ dependencies:
|
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
name: logstash-input-generator
|
91
|
-
type: :development
|
92
91
|
prerelease: false
|
92
|
+
type: :development
|
93
93
|
version_requirements: !ruby/object:Gem::Requirement
|
94
94
|
requirements:
|
95
95
|
- - ">="
|
@@ -102,13 +102,27 @@ dependencies:
|
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: 3.3.11
|
104
104
|
name: ruby-maven
|
105
|
-
type: :development
|
106
105
|
prerelease: false
|
106
|
+
type: :development
|
107
107
|
version_requirements: !ruby/object:Gem::Requirement
|
108
108
|
requirements:
|
109
109
|
- - "~>"
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: 3.3.11
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
name: rspec_junit_formatter
|
119
|
+
prerelease: false
|
120
|
+
type: :development
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
112
126
|
- !ruby/object:Gem::Dependency
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
114
128
|
requirements:
|
@@ -116,8 +130,8 @@ dependencies:
|
|
116
130
|
- !ruby/object:Gem::Version
|
117
131
|
version: '0'
|
118
132
|
name: jar-dependencies
|
119
|
-
type: :runtime
|
120
133
|
prerelease: false
|
134
|
+
type: :runtime
|
121
135
|
version_requirements: !ruby/object:Gem::Requirement
|
122
136
|
requirements:
|
123
137
|
- - ">="
|
@@ -135,38 +149,82 @@ files:
|
|
135
149
|
- Gemfile
|
136
150
|
- LICENSE
|
137
151
|
- README.md
|
138
|
-
-
|
139
|
-
- lib/com/
|
140
|
-
- lib/com/
|
152
|
+
- SECURITY.md
|
153
|
+
- lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar
|
154
|
+
- lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar
|
155
|
+
- lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar
|
156
|
+
- lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar
|
157
|
+
- lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar
|
158
|
+
- lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar
|
159
|
+
- lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar
|
160
|
+
- lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar
|
161
|
+
- lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar
|
162
|
+
- lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar
|
141
163
|
- lib/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar
|
142
|
-
- lib/com/google/code/
|
143
|
-
- lib/com/google/
|
144
|
-
- lib/com/
|
145
|
-
- lib/com/
|
146
|
-
- lib/com/microsoft/azure/azure-
|
147
|
-
- lib/com/microsoft/azure/
|
148
|
-
- lib/com/microsoft/azure/kusto/kusto-
|
164
|
+
- lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar
|
165
|
+
- lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar
|
166
|
+
- lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar
|
167
|
+
- lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar
|
168
|
+
- lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar
|
169
|
+
- lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar
|
170
|
+
- lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar
|
171
|
+
- lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar
|
172
|
+
- lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar
|
173
|
+
- lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar
|
174
|
+
- lib/com/nimbusds/content-type/2.1/content-type-2.1.jar
|
149
175
|
- lib/com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar
|
150
|
-
- lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar
|
151
|
-
- lib/com/nimbusds/oauth2-oidc-sdk/
|
152
|
-
- lib/com/
|
153
|
-
- lib/
|
154
|
-
- lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar
|
176
|
+
- lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar
|
177
|
+
- lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar
|
178
|
+
- lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar
|
179
|
+
- lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar
|
155
180
|
- lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
|
156
|
-
- lib/
|
181
|
+
- lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar
|
182
|
+
- lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar
|
183
|
+
- lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar
|
184
|
+
- lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar
|
185
|
+
- lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar
|
186
|
+
- lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar
|
187
|
+
- lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar
|
188
|
+
- lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar
|
189
|
+
- lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar
|
190
|
+
- lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar
|
191
|
+
- lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar
|
192
|
+
- lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar
|
193
|
+
- lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar
|
194
|
+
- lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar
|
195
|
+
- lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar
|
196
|
+
- lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar
|
197
|
+
- lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar
|
198
|
+
- lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar
|
199
|
+
- lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar
|
200
|
+
- lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar
|
201
|
+
- lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar
|
202
|
+
- lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar
|
203
|
+
- lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar
|
204
|
+
- lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar
|
205
|
+
- lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar
|
206
|
+
- lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar
|
157
207
|
- lib/logstash-output-kusto_jars.rb
|
158
208
|
- lib/logstash/outputs/kusto.rb
|
159
209
|
- lib/logstash/outputs/kusto/ingestor.rb
|
160
210
|
- lib/logstash/outputs/kusto/interval.rb
|
161
|
-
- lib/net/
|
162
|
-
- lib/net/
|
163
|
-
- lib/
|
164
|
-
- lib/
|
165
|
-
- lib/org/apache/
|
166
|
-
- lib/org/
|
167
|
-
- lib/org/
|
168
|
-
- lib/org/
|
211
|
+
- lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar
|
212
|
+
- lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar
|
213
|
+
- lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar
|
214
|
+
- lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar
|
215
|
+
- lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar
|
216
|
+
- lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar
|
217
|
+
- lib/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar
|
218
|
+
- lib/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar
|
219
|
+
- lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar
|
220
|
+
- lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
|
221
|
+
- lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar
|
222
|
+
- lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar
|
223
|
+
- lib/org/json/json/20201115/json-20201115.jar
|
224
|
+
- lib/org/ow2/asm/asm/9.1/asm-9.1.jar
|
225
|
+
- lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar
|
169
226
|
- lib/org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar
|
227
|
+
- lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar
|
170
228
|
- logstash-output-kusto.gemspec
|
171
229
|
- spec/outputs/kusto/ingestor_spec.rb
|
172
230
|
- spec/outputs/kusto_spec.rb
|
@@ -192,8 +250,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
192
250
|
- !ruby/object:Gem::Version
|
193
251
|
version: '0'
|
194
252
|
requirements:
|
195
|
-
- jar 'com.microsoft.azure.kusto, kusto-ingest,
|
196
|
-
rubygems_version: 3.
|
253
|
+
- jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'
|
254
|
+
rubygems_version: 3.2.29
|
197
255
|
signing_key:
|
198
256
|
specification_version: 4
|
199
257
|
summary: Writes events to Azure Data Explorer (Kusto)
|
data/lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar
DELETED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|