logstash-output-kusto 1.0.2-java → 1.0.5-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -1
  3. data/Gemfile +2 -0
  4. data/README.md +15 -0
  5. data/SECURITY.md +41 -0
  6. data/lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar +0 -0
  7. data/lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar +0 -0
  8. data/lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar +0 -0
  9. data/lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar +0 -0
  10. data/lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar +0 -0
  11. data/lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar +0 -0
  12. data/lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar +0 -0
  13. data/lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar +0 -0
  14. data/lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar +0 -0
  15. data/lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar +0 -0
  16. data/lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar +0 -0
  17. data/lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar +0 -0
  18. data/lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar +0 -0
  19. data/lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar +0 -0
  20. data/lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar +0 -0
  21. data/lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar +0 -0
  22. data/lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar +0 -0
  23. data/lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar +0 -0
  24. data/lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar +0 -0
  25. data/lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar +0 -0
  26. data/lib/com/nimbusds/content-type/2.1/content-type-2.1.jar +0 -0
  27. data/lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar +0 -0
  28. data/lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar +0 -0
  29. data/lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar +0 -0
  30. data/lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar +0 -0
  31. data/lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar +0 -0
  32. data/lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar +0 -0
  33. data/lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar +0 -0
  34. data/lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar +0 -0
  35. data/lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar +0 -0
  36. data/lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar +0 -0
  37. data/lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar +0 -0
  38. data/lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar +0 -0
  39. data/lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar +0 -0
  40. data/lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar +0 -0
  41. data/lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar +0 -0
  42. data/lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar +0 -0
  43. data/lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar +0 -0
  44. data/lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar +0 -0
  45. data/lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar +0 -0
  46. data/lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar +0 -0
  47. data/lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar +0 -0
  48. data/lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar +0 -0
  49. data/lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar +0 -0
  50. data/lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar +0 -0
  51. data/lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar +0 -0
  52. data/lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar +0 -0
  53. data/lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar +0 -0
  54. data/lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar +0 -0
  55. data/lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar +0 -0
  56. data/lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar +0 -0
  57. data/lib/logstash/outputs/kusto/ingestor.rb +32 -17
  58. data/lib/logstash/outputs/kusto.rb +11 -2
  59. data/lib/logstash-output-kusto_jars.rb +136 -50
  60. data/lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar +0 -0
  61. data/lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar +0 -0
  62. data/lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar +0 -0
  63. data/lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar +0 -0
  64. data/lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar +0 -0
  65. data/lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar +0 -0
  66. data/lib/org/apache/httpcomponents/httpclient/{4.5.8/httpclient-4.5.8.jar → 4.5.13/httpclient-4.5.13.jar} +0 -0
  67. data/lib/org/apache/httpcomponents/httpcore/{4.4.11/httpcore-4.4.11.jar → 4.4.15/httpcore-4.4.15.jar} +0 -0
  68. data/lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar +0 -0
  69. data/lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar +0 -0
  70. data/lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar +0 -0
  71. data/lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar +0 -0
  72. data/lib/org/json/json/20201115/json-20201115.jar +0 -0
  73. data/lib/org/ow2/asm/asm/9.1/asm-9.1.jar +0 -0
  74. data/lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar +0 -0
  75. data/lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar +0 -0
  76. data/logstash-output-kusto.gemspec +3 -2
  77. data/spec/outputs/kusto/ingestor_spec.rb +19 -7
  78. data/spec/outputs/kusto_spec.rb +7 -5
  79. data/spec/spec_helpers.rb +1 -1
  80. metadata +94 -36
  81. data/lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar +0 -0
  82. data/lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar +0 -0
  83. data/lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar +0 -0
  84. data/lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar +0 -0
  85. data/lib/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
  86. data/lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar +0 -0
  87. data/lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar +0 -0
  88. data/lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar +0 -0
  89. data/lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar +0 -0
  90. data/lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar +0 -0
  91. data/lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar +0 -0
  92. data/lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar +0 -0
  93. data/lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar +0 -0
  94. data/lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar +0 -0
  95. data/lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar +0 -0
  96. data/lib/javax/activation/activation/1.1/activation-1.1.jar +0 -0
  97. data/lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar +0 -0
  98. data/lib/net/minidev/json-smart/2.3/json-smart-2.3.jar +0 -0
  99. data/lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar +0 -0
  100. data/lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar +0 -0
  101. data/lib/org/json/json/20190722/json-20190722.jar +0 -0
  102. data/lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a16109a6fb767545be4b3143fc6f985a7cd81826fd02f76d37465f67465a1f19
4
- data.tar.gz: f5acbf201905cc02addf0f482920b9c321006396fa5838714f8ee53048a6b292
3
+ metadata.gz: 40d2d10177bc99b9da270d95a843ca5490689ac6a9bbdd548744d041e29a557c
4
+ data.tar.gz: b5f85499389d432eb0e60e5be23bb75ea7b9c5c43df6a1d1a7e07f3b5fd44c57
5
5
  SHA512:
6
- metadata.gz: 79be98eb06e2e333b46ab3d0855746f195a09e598f0663242ff111b658c351717565a765af7747077dc95a917040ac095028f9904a3944e66a27162747f79975
7
- data.tar.gz: a3843aeca5467e0f3fa890e01081247fd8257ec2c2b2e2d5f8f2d67c8e988f298812cb2e468d5ab81a1afc26b5272af3efffd4f11977e2cb50487f4de14e3b03
6
+ metadata.gz: 82e08571c9c68c67dd7e2c8f653cfce159b01902cee31982ddefa9cea59769510acb25ebf5d3ae2ed18354a7ccae1ba0dcc6b6135940a706447b017d80d520fb
7
+ data.tar.gz: b3c7c5860bfe0fea8e7da8d5e5356a56b6dd54733214f46c6492e452580c635da637be223a80723db05c2c496edda4d8b607ebe47138a582b0b1fc8be7e5bd4a
data/CHANGELOG.md CHANGED
@@ -28,4 +28,10 @@
28
28
  # 1.0.0
29
29
 
30
30
  - Use stable (2.1.2) version of the java sdk, and retrieve it from maven with bundler.
31
- - Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
31
+ - Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
32
+
33
+
34
+ # 1.0.5
35
+
36
+ - Use (3.1.3) version of the java sdk, and retrieve it from maven with bundler.
37
+ - Added support for `proxy_host` `proxy_port` `proxy_protocol` to support proxying ingestion to Kusto
data/Gemfile CHANGED
@@ -14,6 +14,8 @@ if Dir.exist?(logstash_path) && use_logstash_source
14
14
  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
15
15
  else
16
16
  puts "using default logstash"
17
+ gem 'logstash-core'
18
+ gem 'logstash-core-plugin-api'
17
19
  end
18
20
 
19
21
  gem "rspec"
data/README.md CHANGED
@@ -39,6 +39,9 @@ output {
39
39
  database => "<database name>"
40
40
  table => "<target table>"
41
41
  json_mapping => "<mapping name>"
42
+ proxy_host => "<proxy host>"
43
+ proxy_port => <proxy port>
44
+ proxy_protocol => <"http"|"https">
42
45
  }
43
46
  }
44
47
  ```
@@ -57,6 +60,16 @@ More information about configuring Logstash can be found in the [logstash config
57
60
  | **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files found in the path upon startup | |
58
61
  | **delete_temp_files** | Determines if temp files will be deleted after a successful upload (true is default; set false for debug purposes only)| |
59
62
  | **flush_interval** | The time (in seconds) for flushing writes to temporary files. Default is 2 seconds, 0 will flush on every event. Increase this value to reduce IO calls but keep in mind that events in the buffer will be lost in case of abrupt failure.| |
63
+ | **proxy_host** | The proxy hostname for redirecting traffic to Kusto.| |
64
+ | **proxy_port** | The proxy port for the proxy. Defaults to 80.| |
65
+ | **proxy_protocol** | The proxy server protocol , is one of http or https.| |
66
+
67
+ > Note : LS_JAVA_OPTS can be used to set proxy parameters as well (using export or SET options)
68
+
69
+ ```bash
70
+ export LS_JAVA_OPTS="-Dhttp.proxyHost=1.2.34 -Dhttp.proxyPort=8989 -Dhttps.proxyHost=1.2.3.4 -Dhttps.proxyPort=8989"
71
+ ```
72
+
60
73
 
61
74
  ## Development Requirements
62
75
 
@@ -64,6 +77,8 @@ More information about configuring Logstash can be found in the [logstash config
64
77
  - JRuby 9.2 or higher, defined with openjdk 8 64bit
65
78
  - Logstash, defined with openjdk 8 64bit
66
79
 
80
+ *It is reccomened to use the bundled jdk and jruby with logstash to avoid compatibility issues.*
81
+
67
82
  To fully build the gem, run:
68
83
 
69
84
  ```shell
data/SECURITY.md ADDED
@@ -0,0 +1,41 @@
1
+ <!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
2
+
3
+ ## Security
4
+
5
+ Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
6
+
7
+ If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
8
+
9
+ ## Reporting Security Issues
10
+
11
+ **Please do not report security vulnerabilities through public GitHub issues.**
12
+
13
+ Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
14
+
15
+ If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
16
+
17
+ You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
18
+
19
+ Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
20
+
21
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
22
+ * Full paths of source file(s) related to the manifestation of the issue
23
+ * The location of the affected source code (tag/branch/commit or direct URL)
24
+ * Any special configuration required to reproduce the issue
25
+ * Step-by-step instructions to reproduce the issue
26
+ * Proof-of-concept or exploit code (if possible)
27
+ * Impact of the issue, including how an attacker might exploit the issue
28
+
29
+ This information will help us triage your report more quickly.
30
+
31
+ If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
32
+
33
+ ## Preferred Languages
34
+
35
+ We prefer all communications to be in English.
36
+
37
+ ## Policy
38
+
39
+ Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
40
+
41
+ <!-- END MICROSOFT SECURITY.MD BLOCK -->
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'logstash/outputs/base'
3
+ require 'logstash/outputs/base'
4
4
  require 'logstash/namespace'
5
5
  require 'logstash/errors'
6
6
 
@@ -20,33 +20,40 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
20
20
  LOW_QUEUE_LENGTH = 3
21
21
  FIELD_REF = /%\{[^}]+\}/
22
22
 
23
- def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger, threadpool = DEFAULT_THREADPOOL)
23
+ def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
24
24
  @workers_pool = threadpool
25
25
  @logger = logger
26
-
27
- validate_config(database, table, json_mapping)
28
-
29
- @logger.debug('Preparing Kusto resources.')
26
+ validate_config(database, table, json_mapping,proxy_protocol)
27
+ @logger.info('Preparing Kusto resources.')
30
28
 
31
29
  kusto_java = Java::com.microsoft.azure.kusto
32
- kusto_connection_string = kusto_java.data.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
30
+ apache_http = Java::org.apache.http
31
+ kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
32
+ #
33
33
  @logger.debug(Gem.loaded_specs.to_s)
34
34
  # Unfortunately there's no way to avoid using the gem/plugin name directly...
35
35
  name_for_tracing = "logstash-output-kusto:#{Gem.loaded_specs['logstash-output-kusto']&.version || "unknown"}"
36
36
  @logger.debug("Client name for tracing: #{name_for_tracing}")
37
- kusto_connection_string.setClientVersionForTracing(name_for_tracing)
38
-
39
- @kusto_client = kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
37
+ kusto_connection_string.setClientVersionForTracing(name_for_tracing)
38
+
39
+ @kusto_client = begin
40
+ if proxy_host.nil? || proxy_host.empty?
41
+ kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
42
+ else
43
+ kusto_http_client_properties = kusto_java.data.HttpClientProperties.builder().proxy(apache_http.HttpHost.new(proxy_host,proxy_port,proxy_protocol)).build()
44
+ kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string, kusto_http_client_properties)
45
+ end
46
+ end
40
47
 
41
48
  @ingestion_properties = kusto_java.ingest.IngestionProperties.new(database, table)
42
- @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::Json)
43
- @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DATA_FORMAT::json)
49
+ @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON)
50
+ @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON)
44
51
  @delete_local = delete_local
45
52
 
46
53
  @logger.debug('Kusto resources are ready.')
47
54
  end
48
55
 
49
- def validate_config(database, table, json_mapping)
56
+ def validate_config(database, table, json_mapping,proxy_protocol)
50
57
  if database =~ FIELD_REF
51
58
  @logger.error('database config value should not be dynamic.', database)
52
59
  raise LogStash::ConfigurationError.new('database config value should not be dynamic.')
@@ -61,6 +68,12 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
61
68
  @logger.error('json_mapping config value should not be dynamic.', json_mapping)
62
69
  raise LogStash::ConfigurationError.new('json_mapping config value should not be dynamic.')
63
70
  end
71
+
72
+ if not(["https", "http"].include? proxy_protocol)
73
+ @logger.error('proxy_protocol has to be http or https.', proxy_protocol)
74
+ raise LogStash::ConfigurationError.new('proxy_protocol has to be http or https.')
75
+ end
76
+
64
77
  end
65
78
 
66
79
  def upload_async(path, delete_on_success)
@@ -95,11 +108,13 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
95
108
  # local_ingestion_properties.addJsonMappingName(json_mapping)
96
109
  # end
97
110
 
98
- file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
99
- @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
100
-
111
+ if file_size > 0
112
+ file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
113
+ @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
114
+ else
115
+ @logger.warn("File #{path} is an empty file and is not ingested.")
116
+ end
101
117
  File.delete(path) if delete_on_success
102
-
103
118
  @logger.debug("File #{path} sent to kusto.")
104
119
  rescue Errno::ENOENT => e
105
120
  @logger.error("File doesn't exist! Unrecoverable error.", exception: e.class, message: e.message, path: path, backtrace: e.backtrace)
@@ -88,7 +88,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
88
88
  # Note that this must be in JSON format, as this is the interface between Logstash and Kusto
89
89
  config :json_mapping, validate: :string, required: true
90
90
 
91
- # Mappung name - deprecated, use json_mapping
91
+ # Mapping name - deprecated, use json_mapping
92
92
  config :mapping, validate: :string, deprecated: true
93
93
 
94
94
 
@@ -106,6 +106,15 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
106
106
  # starts processing them in the main thread (not healthy)
107
107
  config :upload_queue_size, validate: :number, default: 30
108
108
 
109
+ # Host of the proxy , is an optional field. Can connect directly
110
+ config :proxy_host, validate: :string, required: false
111
+
112
+ # Port where the proxy runs , defaults to 80. Usually a value like 3128
113
+ config :proxy_port, validate: :number, required: false , default: 80
114
+
115
+ # Check Proxy URL can be over http or https. Dowe need it this way or ignore this & remove this
116
+ config :proxy_protocol, validate: :string, required: false , default: 'http'
117
+
109
118
  default :codec, 'json_lines'
110
119
 
111
120
  def register
@@ -141,7 +150,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
141
150
  max_queue: upload_queue_size,
142
151
  fallback_policy: :caller_runs)
143
152
 
144
- @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, @logger, executor)
153
+ @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor)
145
154
 
146
155
  # send existing files
147
156
  recover_past_files if recovery
@@ -2,63 +2,149 @@
2
2
  begin
3
3
  require 'jar_dependencies'
4
4
  rescue LoadError
5
+ require 'io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar'
6
+ require 'com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar'
7
+ require 'com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar'
8
+ require 'com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar'
9
+ require 'com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar'
10
+ require 'io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar'
11
+ require 'net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar'
12
+ require 'io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar'
13
+ require 'org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar'
14
+ require 'io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar'
15
+ require 'org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar'
16
+ require 'org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar'
17
+ require 'com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar'
18
+ require 'io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar'
19
+ require 'io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar'
20
+ require 'com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar'
21
+ require 'io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar'
22
+ require 'org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar'
23
+ require 'io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar'
24
+ require 'io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar'
25
+ require 'io/vavr/vavr/0.10.2/vavr-0.10.2.jar'
26
+ require 'io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar'
27
+ require 'com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar'
28
+ require 'com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar'
29
+ require 'org/apache/commons/commons-text/1.9/commons-text-1.9.jar'
30
+ require 'net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar'
31
+ require 'com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar'
32
+ require 'io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar'
33
+ require 'jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar'
34
+ require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
35
+ require 'net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar'
36
+ require 'io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar'
37
+ require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
38
+ require 'com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar'
39
+ require 'com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar'
5
40
  require 'com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar'
6
- require 'com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar'
7
- require 'net/minidev/json-smart/2.3/json-smart-2.3.jar'
8
- require 'com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar'
9
- require 'org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar'
10
- require 'com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar'
11
- require 'commons-codec/commons-codec/1.14/commons-codec-1.14.jar'
12
- require 'com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar'
13
- require 'org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar'
14
- require 'com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar'
15
- require 'com/google/code/gson/gson/2.8.0/gson-2.8.0.jar'
16
- require 'javax/activation/activation/1.1/activation-1.1.jar'
17
- require 'net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar'
18
- require 'com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar'
19
- require 'com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar'
20
- require 'org/ow2/asm/asm/5.0.4/asm-5.0.4.jar'
21
- require 'com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar'
22
- require 'com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar'
41
+ require 'com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar'
42
+ require 'io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar'
43
+ require 'io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar'
44
+ require 'com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar'
45
+ require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
46
+ require 'org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar'
47
+ require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
48
+ require 'com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar'
49
+ require 'io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar'
50
+ require 'com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar'
51
+ require 'org/ow2/asm/asm/9.1/asm-9.1.jar'
52
+ require 'com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar'
53
+ require 'com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar'
54
+ require 'org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar'
55
+ require 'org/json/json/20201115/json-20201115.jar'
56
+ require 'io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar'
57
+ require 'io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar'
58
+ require 'net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar'
59
+ require 'io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar'
60
+ require 'io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar'
61
+ require 'jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar'
62
+ require 'io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar'
63
+ require 'com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar'
64
+ require 'com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar'
65
+ require 'io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar'
66
+ require 'com/nimbusds/content-type/2.1/content-type-2.1.jar'
23
67
  require 'org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar'
24
- require 'org/apache/httpcomponents/httpcore/4.4.11/httpcore-4.4.11.jar'
68
+ require 'com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar'
25
69
  require 'com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar'
26
- require 'com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar'
27
- require 'com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar'
28
- require 'com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar'
29
- require 'com/google/guava/guava/20.0/guava-20.0.jar'
30
- require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
31
- require 'org/json/json/20190722/json-20190722.jar'
32
- require 'org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar'
70
+ require 'com/azure/azure-core/1.21.0/azure-core-1.21.0.jar'
71
+ require 'com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar'
72
+ require 'org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar'
73
+ require 'io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar'
74
+ require 'org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar'
75
+ require 'io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar'
33
76
  end
34
77
 
35
78
  if defined? Jars
79
+ require_jar 'io.netty', 'netty-resolver-dns-native-macos', 'osx-x86_64', '4.1.68.Final'
80
+ require_jar 'com.google.guava', 'guava', '24.1.1-jre'
81
+ require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.1.3'
82
+ require_jar 'com.microsoft.azure', 'azure-storage', '8.6.6'
83
+ require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.1.3'
84
+ require_jar 'io.projectreactor.netty', 'reactor-netty-http', '1.0.11'
85
+ require_jar 'net.java.dev.jna', 'jna', '5.5.0'
86
+ require_jar 'io.netty', 'netty-handler-proxy', '4.1.68.Final'
87
+ require_jar 'org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'
88
+ require_jar 'io.netty', 'netty-codec-socks', '4.1.68.Final'
89
+ require_jar 'org.apache.commons', 'commons-lang3', '3.11'
90
+ require_jar 'org.reactivestreams', 'reactive-streams', '1.0.3'
91
+ require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.2.4'
92
+ require_jar 'io.github.resilience4j', 'resilience4j-core', '1.7.1'
93
+ require_jar 'io.netty', 'netty-codec-dns', '4.1.68.Final'
94
+ require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.9.3'
95
+ require_jar 'io.netty', 'netty-transport', '4.1.68.Final'
96
+ require_jar 'org.jetbrains', 'annotations', '22.0.0'
97
+ require_jar 'io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'
98
+ require_jar 'io.netty', 'netty-transport-native-epoll', 'linux-x86_64', '4.1.68.Final'
99
+ require_jar 'io.vavr', 'vavr', '0.10.2'
100
+ require_jar 'io.netty', 'netty-resolver-dns', '4.1.68.Final'
101
+ require_jar 'com.azure', 'azure-identity', '1.3.7'
102
+ require_jar 'com.fasterxml.jackson.module', 'jackson-module-jaxb-annotations', '2.12.5'
103
+ require_jar 'org.apache.commons', 'commons-text', '1.9'
104
+ require_jar 'net.java.dev.jna', 'jna-platform', '5.6.0'
105
+ require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.13.0'
106
+ require_jar 'io.netty', 'netty-resolver', '4.1.68.Final'
107
+ require_jar 'jakarta.activation', 'jakarta.activation-api', '1.2.1'
108
+ require_jar 'commons-logging', 'commons-logging', '1.2'
109
+ require_jar 'net.minidev', 'accessors-smart', '2.4.7'
110
+ require_jar 'io.netty', 'netty-codec', '4.1.68.Final'
111
+ require_jar 'commons-codec', 'commons-codec', '1.11'
112
+ require_jar 'com.microsoft.azure', 'msal4j-persistence-extension', '1.1.0'
113
+ require_jar 'com.fasterxml.woodstox', 'woodstox-core', '6.2.4'
36
114
  require_jar 'com.github.stephenc.jcip', 'jcip-annotations', '1.0-1'
37
- require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.3'
38
- require_jar 'net.minidev', 'json-smart', '2.3'
39
- require_jar 'com.microsoft.azure.kusto', 'kusto-data', '2.1.2'
40
- require_jar 'org.apache.commons', 'commons-lang3', '3.9'
41
- require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.7'
42
- require_jar 'commons-codec', 'commons-codec', '1.14'
43
- require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '2.1.2'
44
- require_jar 'org.jetbrains', 'annotations', '17.0.0'
45
- require_jar 'com.microsoft.azure', 'azure-storage', '8.3.0'
46
- require_jar 'com.google.code.gson', 'gson', '2.8.0'
47
- require_jar 'javax.activation', 'activation', '1.1'
48
- require_jar 'net.minidev', 'accessors-smart', '1.2'
49
- require_jar 'com.microsoft.azure', 'adal4j', '1.6.5'
50
- require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10'
51
- require_jar 'org.ow2.asm', 'asm', '5.0.4'
52
- require_jar 'com.sun.mail', 'javax.mail', '1.6.1'
53
- require_jar 'com.nimbusds', 'oauth2-oidc-sdk', '6.5'
115
+ require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.13.0'
116
+ require_jar 'io.netty', 'netty-transport-native-kqueue', 'osx-x86_64', '4.1.68.Final'
117
+ require_jar 'io.netty', 'netty-handler', '4.1.68.Final'
118
+ require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.13.0'
119
+ require_jar 'com.google.code.findbugs', 'jsr305', '1.3.9'
120
+ require_jar 'org.checkerframework', 'checker-compat-qual', '2.0.0'
121
+ require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
122
+ require_jar 'com.google.j2objc', 'j2objc-annotations', '1.1'
123
+ require_jar 'io.netty', 'netty-codec-http', '4.1.68.Final'
124
+ require_jar 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-xml', '2.12.5'
125
+ require_jar 'org.ow2.asm', 'asm', '9.1'
126
+ require_jar 'com.azure', 'azure-core-http-netty', '1.11.1'
127
+ require_jar 'com.fasterxml.jackson.datatype', 'jackson-datatype-jsr310', '2.12.5'
128
+ require_jar 'org.codehaus.woodstox', 'stax2-api', '4.2.1'
129
+ require_jar 'org.json', 'json', '20201115'
130
+ require_jar 'io.netty', 'netty-tcnative-boringssl-static', '2.0.43.Final'
131
+ require_jar 'io.netty', 'netty-common', '4.1.68.Final'
132
+ require_jar 'net.minidev', 'json-smart', '2.4.7'
133
+ require_jar 'io.projectreactor', 'reactor-core', '3.4.10'
134
+ require_jar 'io.netty', 'netty-buffer', '4.1.68.Final'
135
+ require_jar 'jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.2'
136
+ require_jar 'io.github.resilience4j', 'resilience4j-retry', '1.7.1'
137
+ require_jar 'com.google.errorprone', 'error_prone_annotations', '2.1.3'
138
+ require_jar 'com.univocity', 'univocity-parsers', '2.9.1'
139
+ require_jar 'io.vavr', 'vavr-match', '0.10.2'
140
+ require_jar 'com.nimbusds', 'content-type', '2.1'
54
141
  require_jar 'org.slf4j', 'slf4j-api', '1.8.0-beta4'
55
- require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.11'
142
+ require_jar 'com.microsoft.azure', 'msal4j', '1.11.0'
56
143
  require_jar 'com.nimbusds', 'lang-tag', '1.5'
57
- require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.9.4'
58
- require_jar 'com.univocity', 'univocity-parsers', '2.1.1'
59
- require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.0.0'
60
- require_jar 'com.google.guava', 'guava', '20.0'
61
- require_jar 'commons-logging', 'commons-logging', '1.2'
62
- require_jar 'org.json', 'json', '20190722'
63
- require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.8'
144
+ require_jar 'com.azure', 'azure-core', '1.21.0'
145
+ require_jar 'com.nimbusds', 'oauth2-oidc-sdk', '9.7'
146
+ require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.15'
147
+ require_jar 'io.netty', 'netty-codec-http2', '4.1.68.Final'
148
+ require_jar 'org.slf4j', 'slf4j-simple', '1.8.0-beta4'
149
+ require_jar 'io.projectreactor.netty', 'reactor-netty-core', '1.0.11'
64
150
  end
Binary file
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
3
- s.version = '1.0.2'
3
+ s.version = '1.0.5'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Writes events to Azure Data Explorer (Kusto)'
6
6
  s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
@@ -28,8 +28,9 @@ Gem::Specification.new do |s|
28
28
  s.add_development_dependency 'flores'
29
29
  s.add_development_dependency 'logstash-input-generator'
30
30
  s.add_development_dependency 'ruby-maven', '~> 3.3.11'
31
+ s.add_development_dependency 'rspec_junit_formatter'
31
32
 
32
33
  # Jar dependencies
33
- s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2"
34
+ s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'"
34
35
  s.add_runtime_dependency 'jar-dependencies'
35
36
  end
@@ -5,12 +5,15 @@ require 'logstash/outputs/kusto/ingestor'
5
5
 
6
6
  describe LogStash::Outputs::Kusto::Ingestor do
7
7
 
8
- let(:ingest_url) { "mycluster" }
8
+ let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
9
9
  let(:app_id) { "myid" }
10
10
  let(:app_key) { LogStash::Util::Password.new("mykey") }
11
11
  let(:app_tenant) { "mytenant" }
12
12
  let(:database) { "mydatabase" }
13
13
  let(:table) { "mytable" }
14
+ let(:proxy_host) { "localhost" }
15
+ let(:proxy_port) { 80 }
16
+ let(:proxy_protocol) { "http" }
14
17
  let(:json_mapping) { "mymapping" }
15
18
  let(:delete_local) { false }
16
19
  let(:logger) { spy('logger') }
@@ -21,7 +24,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
21
24
  # note that this will cause an internal error since connection is being tried.
22
25
  # however we still want to test that all the java stuff is working as expected
23
26
  expect {
24
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger)
27
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
25
28
  ingestor.stop
26
29
  }.not_to raise_error
27
30
  end
@@ -32,7 +35,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
32
35
  dynamic_name_array.each do |test_database|
33
36
  it "with database: #{test_database}" do
34
37
  expect {
35
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, logger)
38
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
36
39
  ingestor.stop
37
40
  }.to raise_error(LogStash::ConfigurationError)
38
41
  end
@@ -43,7 +46,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
43
46
  dynamic_name_array.each do |test_table|
44
47
  it "with database: #{test_table}" do
45
48
  expect {
46
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, logger)
49
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
47
50
  ingestor.stop
48
51
  }.to raise_error(LogStash::ConfigurationError)
49
52
  end
@@ -51,16 +54,25 @@ describe LogStash::Outputs::Kusto::Ingestor do
51
54
  end
52
55
 
53
56
  context 'doesnt allow mapping to have some dynamic part' do
54
- dynamic_name_array.each do |test_json_mapping|
55
- it "with database: #{test_json_mapping}" do
57
+ dynamic_name_array.each do |json_mapping|
58
+ it "with database: #{json_mapping}" do
56
59
  expect {
57
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, test_json_mapping, delete_local, logger)
60
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
58
61
  ingestor.stop
59
62
  }.to raise_error(LogStash::ConfigurationError)
60
63
  end
61
64
  end
62
65
  end
63
66
 
67
+ context 'proxy protocol has to be http or https' do
68
+ it "with proxy protocol: socks" do
69
+ expect {
70
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
71
+ ingestor.stop
72
+ }.to raise_error(LogStash::ConfigurationError)
73
+ end
74
+ end
75
+
64
76
  end
65
77
 
66
78
  # describe 'receiving events' do
@@ -1,5 +1,4 @@
1
1
  # encoding: utf-8
2
- require 'logstash/devutils/rspec/spec_helper'
3
2
  require 'logstash/outputs/kusto'
4
3
  require 'logstash/codecs/plain'
5
4
  require 'logstash/event'
@@ -7,13 +6,16 @@ require 'logstash/event'
7
6
  describe LogStash::Outputs::Kusto do
8
7
 
9
8
  let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
10
- "ingest_url" => "mycluster",
9
+ "ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
11
10
  "app_id" => "myid",
12
11
  "app_key" => "mykey",
13
- "app_tenant" => "mytenant",
12
+ "app_tenant" => "mytenant",
14
13
  "database" => "mydatabase",
15
14
  "table" => "mytable",
16
- "json_mapping" => "mymapping"
15
+ "json_mapping" => "mymapping",
16
+ "proxy_host" => "localhost",
17
+ "proxy_port" => 3128,
18
+ "proxy_protocol" => "https"
17
19
  } }
18
20
 
19
21
  describe '#register' do
@@ -50,5 +52,5 @@ describe LogStash::Outputs::Kusto do
50
52
  end
51
53
 
52
54
  end
53
-
55
+
54
56
  end
data/spec/spec_helpers.rb CHANGED
@@ -18,4 +18,4 @@ RSpec.configure do |config|
18
18
  $stdout = STDOUT
19
19
  $stderr = STDERR
20
20
  end
21
- end
21
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-kusto
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.5
5
5
  platform: java
6
6
  authors:
7
7
  - Tamir Kamara
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-12-27 00:00:00.000000000 Z
12
+ date: 2022-08-04 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -18,8 +18,8 @@ dependencies:
18
18
  - !ruby/object:Gem::Version
19
19
  version: '2.0'
20
20
  name: logstash-core-plugin-api
21
- type: :runtime
22
21
  prerelease: false
22
+ type: :runtime
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - "~>"
@@ -32,8 +32,8 @@ dependencies:
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  name: logstash-codec-json_lines
35
- type: :runtime
36
35
  prerelease: false
36
+ type: :runtime
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - ">="
@@ -46,8 +46,8 @@ dependencies:
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  name: logstash-codec-line
49
- type: :runtime
50
49
  prerelease: false
50
+ type: :runtime
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - ">="
@@ -60,8 +60,8 @@ dependencies:
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  name: logstash-devutils
63
- type: :development
64
63
  prerelease: false
64
+ type: :development
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
67
  - - ">="
@@ -74,8 +74,8 @@ dependencies:
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  name: flores
77
- type: :development
78
77
  prerelease: false
78
+ type: :development
79
79
  version_requirements: !ruby/object:Gem::Requirement
80
80
  requirements:
81
81
  - - ">="
@@ -88,8 +88,8 @@ dependencies:
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  name: logstash-input-generator
91
- type: :development
92
91
  prerelease: false
92
+ type: :development
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
95
  - - ">="
@@ -102,13 +102,27 @@ dependencies:
102
102
  - !ruby/object:Gem::Version
103
103
  version: 3.3.11
104
104
  name: ruby-maven
105
- type: :development
106
105
  prerelease: false
106
+ type: :development
107
107
  version_requirements: !ruby/object:Gem::Requirement
108
108
  requirements:
109
109
  - - "~>"
110
110
  - !ruby/object:Gem::Version
111
111
  version: 3.3.11
112
+ - !ruby/object:Gem::Dependency
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ name: rspec_junit_formatter
119
+ prerelease: false
120
+ type: :development
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - ">="
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
112
126
  - !ruby/object:Gem::Dependency
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
@@ -116,8 +130,8 @@ dependencies:
116
130
  - !ruby/object:Gem::Version
117
131
  version: '0'
118
132
  name: jar-dependencies
119
- type: :runtime
120
133
  prerelease: false
134
+ type: :runtime
121
135
  version_requirements: !ruby/object:Gem::Requirement
122
136
  requirements:
123
137
  - - ">="
@@ -135,38 +149,82 @@ files:
135
149
  - Gemfile
136
150
  - LICENSE
137
151
  - README.md
138
- - lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar
139
- - lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar
140
- - lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar
152
+ - SECURITY.md
153
+ - lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar
154
+ - lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar
155
+ - lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar
156
+ - lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar
157
+ - lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar
158
+ - lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar
159
+ - lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar
160
+ - lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar
161
+ - lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar
162
+ - lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar
141
163
  - lib/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar
142
- - lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar
143
- - lib/com/google/guava/guava/20.0/guava-20.0.jar
144
- - lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar
145
- - lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar
146
- - lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar
147
- - lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar
148
- - lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar
164
+ - lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar
165
+ - lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar
166
+ - lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar
167
+ - lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar
168
+ - lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar
169
+ - lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar
170
+ - lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar
171
+ - lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar
172
+ - lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar
173
+ - lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar
174
+ - lib/com/nimbusds/content-type/2.1/content-type-2.1.jar
149
175
  - lib/com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar
150
- - lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar
151
- - lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar
152
- - lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar
153
- - lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar
154
- - lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar
176
+ - lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar
177
+ - lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar
178
+ - lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar
179
+ - lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar
155
180
  - lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
156
- - lib/javax/activation/activation/1.1/activation-1.1.jar
181
+ - lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar
182
+ - lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar
183
+ - lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar
184
+ - lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar
185
+ - lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar
186
+ - lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar
187
+ - lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar
188
+ - lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar
189
+ - lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar
190
+ - lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar
191
+ - lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar
192
+ - lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar
193
+ - lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar
194
+ - lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar
195
+ - lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar
196
+ - lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar
197
+ - lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar
198
+ - lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar
199
+ - lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar
200
+ - lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar
201
+ - lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar
202
+ - lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar
203
+ - lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar
204
+ - lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar
205
+ - lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar
206
+ - lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar
157
207
  - lib/logstash-output-kusto_jars.rb
158
208
  - lib/logstash/outputs/kusto.rb
159
209
  - lib/logstash/outputs/kusto/ingestor.rb
160
210
  - lib/logstash/outputs/kusto/interval.rb
161
- - lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar
162
- - lib/net/minidev/json-smart/2.3/json-smart-2.3.jar
163
- - lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar
164
- - lib/org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar
165
- - lib/org/apache/httpcomponents/httpcore/4.4.11/httpcore-4.4.11.jar
166
- - lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar
167
- - lib/org/json/json/20190722/json-20190722.jar
168
- - lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar
211
+ - lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar
212
+ - lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar
213
+ - lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar
214
+ - lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar
215
+ - lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar
216
+ - lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar
217
+ - lib/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar
218
+ - lib/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar
219
+ - lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar
220
+ - lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
221
+ - lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar
222
+ - lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar
223
+ - lib/org/json/json/20201115/json-20201115.jar
224
+ - lib/org/ow2/asm/asm/9.1/asm-9.1.jar
225
+ - lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar
169
226
  - lib/org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar
227
+ - lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar
170
228
  - logstash-output-kusto.gemspec
171
229
  - spec/outputs/kusto/ingestor_spec.rb
172
230
  - spec/outputs/kusto_spec.rb
@@ -192,8 +250,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
192
250
  - !ruby/object:Gem::Version
193
251
  version: '0'
194
252
  requirements:
195
- - jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2
196
- rubygems_version: 3.0.6
253
+ - jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'
254
+ rubygems_version: 3.2.29
197
255
  signing_key:
198
256
  specification_version: 4
199
257
  summary: Writes events to Azure Data Explorer (Kusto)