logstash-output-kusto 1.0.2-java → 1.0.5-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -1
  3. data/Gemfile +2 -0
  4. data/README.md +15 -0
  5. data/SECURITY.md +41 -0
  6. data/lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar +0 -0
  7. data/lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar +0 -0
  8. data/lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar +0 -0
  9. data/lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar +0 -0
  10. data/lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar +0 -0
  11. data/lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar +0 -0
  12. data/lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar +0 -0
  13. data/lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar +0 -0
  14. data/lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar +0 -0
  15. data/lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar +0 -0
  16. data/lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar +0 -0
  17. data/lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar +0 -0
  18. data/lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar +0 -0
  19. data/lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar +0 -0
  20. data/lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar +0 -0
  21. data/lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar +0 -0
  22. data/lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar +0 -0
  23. data/lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar +0 -0
  24. data/lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar +0 -0
  25. data/lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar +0 -0
  26. data/lib/com/nimbusds/content-type/2.1/content-type-2.1.jar +0 -0
  27. data/lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar +0 -0
  28. data/lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar +0 -0
  29. data/lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar +0 -0
  30. data/lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar +0 -0
  31. data/lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar +0 -0
  32. data/lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar +0 -0
  33. data/lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar +0 -0
  34. data/lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar +0 -0
  35. data/lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar +0 -0
  36. data/lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar +0 -0
  37. data/lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar +0 -0
  38. data/lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar +0 -0
  39. data/lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar +0 -0
  40. data/lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar +0 -0
  41. data/lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar +0 -0
  42. data/lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar +0 -0
  43. data/lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar +0 -0
  44. data/lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar +0 -0
  45. data/lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar +0 -0
  46. data/lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar +0 -0
  47. data/lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar +0 -0
  48. data/lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar +0 -0
  49. data/lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar +0 -0
  50. data/lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar +0 -0
  51. data/lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar +0 -0
  52. data/lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar +0 -0
  53. data/lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar +0 -0
  54. data/lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar +0 -0
  55. data/lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar +0 -0
  56. data/lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar +0 -0
  57. data/lib/logstash/outputs/kusto/ingestor.rb +32 -17
  58. data/lib/logstash/outputs/kusto.rb +11 -2
  59. data/lib/logstash-output-kusto_jars.rb +136 -50
  60. data/lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar +0 -0
  61. data/lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar +0 -0
  62. data/lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar +0 -0
  63. data/lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar +0 -0
  64. data/lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar +0 -0
  65. data/lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar +0 -0
  66. data/lib/org/apache/httpcomponents/httpclient/{4.5.8/httpclient-4.5.8.jar → 4.5.13/httpclient-4.5.13.jar} +0 -0
  67. data/lib/org/apache/httpcomponents/httpcore/{4.4.11/httpcore-4.4.11.jar → 4.4.15/httpcore-4.4.15.jar} +0 -0
  68. data/lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar +0 -0
  69. data/lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar +0 -0
  70. data/lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar +0 -0
  71. data/lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar +0 -0
  72. data/lib/org/json/json/20201115/json-20201115.jar +0 -0
  73. data/lib/org/ow2/asm/asm/9.1/asm-9.1.jar +0 -0
  74. data/lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar +0 -0
  75. data/lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar +0 -0
  76. data/logstash-output-kusto.gemspec +3 -2
  77. data/spec/outputs/kusto/ingestor_spec.rb +19 -7
  78. data/spec/outputs/kusto_spec.rb +7 -5
  79. data/spec/spec_helpers.rb +1 -1
  80. metadata +94 -36
  81. data/lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar +0 -0
  82. data/lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar +0 -0
  83. data/lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar +0 -0
  84. data/lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar +0 -0
  85. data/lib/com/google/guava/guava/20.0/guava-20.0.jar +0 -0
  86. data/lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar +0 -0
  87. data/lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar +0 -0
  88. data/lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar +0 -0
  89. data/lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar +0 -0
  90. data/lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar +0 -0
  91. data/lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar +0 -0
  92. data/lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar +0 -0
  93. data/lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar +0 -0
  94. data/lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar +0 -0
  95. data/lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar +0 -0
  96. data/lib/javax/activation/activation/1.1/activation-1.1.jar +0 -0
  97. data/lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar +0 -0
  98. data/lib/net/minidev/json-smart/2.3/json-smart-2.3.jar +0 -0
  99. data/lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar +0 -0
  100. data/lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar +0 -0
  101. data/lib/org/json/json/20190722/json-20190722.jar +0 -0
  102. data/lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a16109a6fb767545be4b3143fc6f985a7cd81826fd02f76d37465f67465a1f19
4
- data.tar.gz: f5acbf201905cc02addf0f482920b9c321006396fa5838714f8ee53048a6b292
3
+ metadata.gz: 40d2d10177bc99b9da270d95a843ca5490689ac6a9bbdd548744d041e29a557c
4
+ data.tar.gz: b5f85499389d432eb0e60e5be23bb75ea7b9c5c43df6a1d1a7e07f3b5fd44c57
5
5
  SHA512:
6
- metadata.gz: 79be98eb06e2e333b46ab3d0855746f195a09e598f0663242ff111b658c351717565a765af7747077dc95a917040ac095028f9904a3944e66a27162747f79975
7
- data.tar.gz: a3843aeca5467e0f3fa890e01081247fd8257ec2c2b2e2d5f8f2d67c8e988f298812cb2e468d5ab81a1afc26b5272af3efffd4f11977e2cb50487f4de14e3b03
6
+ metadata.gz: 82e08571c9c68c67dd7e2c8f653cfce159b01902cee31982ddefa9cea59769510acb25ebf5d3ae2ed18354a7ccae1ba0dcc6b6135940a706447b017d80d520fb
7
+ data.tar.gz: b3c7c5860bfe0fea8e7da8d5e5356a56b6dd54733214f46c6492e452580c635da637be223a80723db05c2c496edda4d8b607ebe47138a582b0b1fc8be7e5bd4a
data/CHANGELOG.md CHANGED
@@ -28,4 +28,10 @@
28
28
  # 1.0.0
29
29
 
30
30
  - Use stable (2.1.2) version of the java sdk, and retrieve it from maven with bundler.
31
- - Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
31
+ - Renamed `mapping` to `json_mapping` in order to clarify usage. `mapping` still remains as a deprecated parameter.
32
+
33
+
34
+ # 1.0.5
35
+
36
+ - Use (3.1.3) version of the java sdk, and retrieve it from maven with bundler.
37
+ - Added support for `proxy_host` `proxy_port` `proxy_protocol` to support proxying ingestion to Kusto
data/Gemfile CHANGED
@@ -14,6 +14,8 @@ if Dir.exist?(logstash_path) && use_logstash_source
14
14
  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
15
15
  else
16
16
  puts "using default logstash"
17
+ gem 'logstash-core'
18
+ gem 'logstash-core-plugin-api'
17
19
  end
18
20
 
19
21
  gem "rspec"
data/README.md CHANGED
@@ -39,6 +39,9 @@ output {
39
39
  database => "<database name>"
40
40
  table => "<target table>"
41
41
  json_mapping => "<mapping name>"
42
+ proxy_host => "<proxy host>"
43
+ proxy_port => <proxy port>
44
+ proxy_protocol => <"http"|"https">
42
45
  }
43
46
  }
44
47
  ```
@@ -57,6 +60,16 @@ More information about configuring Logstash can be found in the [logstash config
57
60
  | **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files found in the path upon startup | |
58
61
  | **delete_temp_files** | Determines if temp files will be deleted after a successful upload (true is default; set false for debug purposes only)| |
59
62
  | **flush_interval** | The time (in seconds) for flushing writes to temporary files. Default is 2 seconds, 0 will flush on every event. Increase this value to reduce IO calls but keep in mind that events in the buffer will be lost in case of abrupt failure.| |
63
+ | **proxy_host** | The proxy hostname for redirecting traffic to Kusto.| |
64
+ | **proxy_port** | The proxy port for the proxy. Defaults to 80.| |
65
+ | **proxy_protocol** | The proxy server protocol , is one of http or https.| |
66
+
67
+ > Note : LS_JAVA_OPTS can be used to set proxy parameters as well (using export or SET options)
68
+
69
+ ```bash
70
+ export LS_JAVA_OPTS="-Dhttp.proxyHost=1.2.34 -Dhttp.proxyPort=8989 -Dhttps.proxyHost=1.2.3.4 -Dhttps.proxyPort=8989"
71
+ ```
72
+
60
73
 
61
74
  ## Development Requirements
62
75
 
@@ -64,6 +77,8 @@ More information about configuring Logstash can be found in the [logstash config
64
77
  - JRuby 9.2 or higher, defined with openjdk 8 64bit
65
78
  - Logstash, defined with openjdk 8 64bit
66
79
 
80
+ *It is reccomened to use the bundled jdk and jruby with logstash to avoid compatibility issues.*
81
+
67
82
  To fully build the gem, run:
68
83
 
69
84
  ```shell
data/SECURITY.md ADDED
@@ -0,0 +1,41 @@
1
+ <!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
2
+
3
+ ## Security
4
+
5
+ Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
6
+
7
+ If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
8
+
9
+ ## Reporting Security Issues
10
+
11
+ **Please do not report security vulnerabilities through public GitHub issues.**
12
+
13
+ Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
14
+
15
+ If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
16
+
17
+ You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
18
+
19
+ Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
20
+
21
+ * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
22
+ * Full paths of source file(s) related to the manifestation of the issue
23
+ * The location of the affected source code (tag/branch/commit or direct URL)
24
+ * Any special configuration required to reproduce the issue
25
+ * Step-by-step instructions to reproduce the issue
26
+ * Proof-of-concept or exploit code (if possible)
27
+ * Impact of the issue, including how an attacker might exploit the issue
28
+
29
+ This information will help us triage your report more quickly.
30
+
31
+ If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
32
+
33
+ ## Preferred Languages
34
+
35
+ We prefer all communications to be in English.
36
+
37
+ ## Policy
38
+
39
+ Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
40
+
41
+ <!-- END MICROSOFT SECURITY.MD BLOCK -->
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
 
3
- require 'logstash/outputs/base'
3
+ require 'logstash/outputs/base'
4
4
  require 'logstash/namespace'
5
5
  require 'logstash/errors'
6
6
 
@@ -20,33 +20,40 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
20
20
  LOW_QUEUE_LENGTH = 3
21
21
  FIELD_REF = /%\{[^}]+\}/
22
22
 
23
- def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger, threadpool = DEFAULT_THREADPOOL)
23
+ def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
24
24
  @workers_pool = threadpool
25
25
  @logger = logger
26
-
27
- validate_config(database, table, json_mapping)
28
-
29
- @logger.debug('Preparing Kusto resources.')
26
+ validate_config(database, table, json_mapping,proxy_protocol)
27
+ @logger.info('Preparing Kusto resources.')
30
28
 
31
29
  kusto_java = Java::com.microsoft.azure.kusto
32
- kusto_connection_string = kusto_java.data.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
30
+ apache_http = Java::org.apache.http
31
+ kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
32
+ #
33
33
  @logger.debug(Gem.loaded_specs.to_s)
34
34
  # Unfortunately there's no way to avoid using the gem/plugin name directly...
35
35
  name_for_tracing = "logstash-output-kusto:#{Gem.loaded_specs['logstash-output-kusto']&.version || "unknown"}"
36
36
  @logger.debug("Client name for tracing: #{name_for_tracing}")
37
- kusto_connection_string.setClientVersionForTracing(name_for_tracing)
38
-
39
- @kusto_client = kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
37
+ kusto_connection_string.setClientVersionForTracing(name_for_tracing)
38
+
39
+ @kusto_client = begin
40
+ if proxy_host.nil? || proxy_host.empty?
41
+ kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
42
+ else
43
+ kusto_http_client_properties = kusto_java.data.HttpClientProperties.builder().proxy(apache_http.HttpHost.new(proxy_host,proxy_port,proxy_protocol)).build()
44
+ kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string, kusto_http_client_properties)
45
+ end
46
+ end
40
47
 
41
48
  @ingestion_properties = kusto_java.ingest.IngestionProperties.new(database, table)
42
- @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::Json)
43
- @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DATA_FORMAT::json)
49
+ @ingestion_properties.setIngestionMapping(json_mapping, kusto_java.ingest.IngestionMapping::IngestionMappingKind::JSON)
50
+ @ingestion_properties.setDataFormat(kusto_java.ingest.IngestionProperties::DataFormat::JSON)
44
51
  @delete_local = delete_local
45
52
 
46
53
  @logger.debug('Kusto resources are ready.')
47
54
  end
48
55
 
49
- def validate_config(database, table, json_mapping)
56
+ def validate_config(database, table, json_mapping,proxy_protocol)
50
57
  if database =~ FIELD_REF
51
58
  @logger.error('database config value should not be dynamic.', database)
52
59
  raise LogStash::ConfigurationError.new('database config value should not be dynamic.')
@@ -61,6 +68,12 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
61
68
  @logger.error('json_mapping config value should not be dynamic.', json_mapping)
62
69
  raise LogStash::ConfigurationError.new('json_mapping config value should not be dynamic.')
63
70
  end
71
+
72
+ if not(["https", "http"].include? proxy_protocol)
73
+ @logger.error('proxy_protocol has to be http or https.', proxy_protocol)
74
+ raise LogStash::ConfigurationError.new('proxy_protocol has to be http or https.')
75
+ end
76
+
64
77
  end
65
78
 
66
79
  def upload_async(path, delete_on_success)
@@ -95,11 +108,13 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
95
108
  # local_ingestion_properties.addJsonMappingName(json_mapping)
96
109
  # end
97
110
 
98
- file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
99
- @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
100
-
111
+ if file_size > 0
112
+ file_source_info = Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, 0); # 0 - let the sdk figure out the size of the file
113
+ @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
114
+ else
115
+ @logger.warn("File #{path} is an empty file and is not ingested.")
116
+ end
101
117
  File.delete(path) if delete_on_success
102
-
103
118
  @logger.debug("File #{path} sent to kusto.")
104
119
  rescue Errno::ENOENT => e
105
120
  @logger.error("File doesn't exist! Unrecoverable error.", exception: e.class, message: e.message, path: path, backtrace: e.backtrace)
@@ -88,7 +88,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
88
88
  # Note that this must be in JSON format, as this is the interface between Logstash and Kusto
89
89
  config :json_mapping, validate: :string, required: true
90
90
 
91
- # Mappung name - deprecated, use json_mapping
91
+ # Mapping name - deprecated, use json_mapping
92
92
  config :mapping, validate: :string, deprecated: true
93
93
 
94
94
 
@@ -106,6 +106,15 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
106
106
  # starts processing them in the main thread (not healthy)
107
107
  config :upload_queue_size, validate: :number, default: 30
108
108
 
109
+ # Host of the proxy , is an optional field. Can connect directly
110
+ config :proxy_host, validate: :string, required: false
111
+
112
+ # Port where the proxy runs , defaults to 80. Usually a value like 3128
113
+ config :proxy_port, validate: :number, required: false , default: 80
114
+
115
+ # Check Proxy URL can be over http or https. Dowe need it this way or ignore this & remove this
116
+ config :proxy_protocol, validate: :string, required: false , default: 'http'
117
+
109
118
  default :codec, 'json_lines'
110
119
 
111
120
  def register
@@ -141,7 +150,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
141
150
  max_queue: upload_queue_size,
142
151
  fallback_policy: :caller_runs)
143
152
 
144
- @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, @logger, executor)
153
+ @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor)
145
154
 
146
155
  # send existing files
147
156
  recover_past_files if recovery
@@ -2,63 +2,149 @@
2
2
  begin
3
3
  require 'jar_dependencies'
4
4
  rescue LoadError
5
+ require 'io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar'
6
+ require 'com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar'
7
+ require 'com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar'
8
+ require 'com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar'
9
+ require 'com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar'
10
+ require 'io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar'
11
+ require 'net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar'
12
+ require 'io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar'
13
+ require 'org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar'
14
+ require 'io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar'
15
+ require 'org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar'
16
+ require 'org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar'
17
+ require 'com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar'
18
+ require 'io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar'
19
+ require 'io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar'
20
+ require 'com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar'
21
+ require 'io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar'
22
+ require 'org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar'
23
+ require 'io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar'
24
+ require 'io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar'
25
+ require 'io/vavr/vavr/0.10.2/vavr-0.10.2.jar'
26
+ require 'io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar'
27
+ require 'com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar'
28
+ require 'com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar'
29
+ require 'org/apache/commons/commons-text/1.9/commons-text-1.9.jar'
30
+ require 'net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar'
31
+ require 'com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar'
32
+ require 'io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar'
33
+ require 'jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar'
34
+ require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
35
+ require 'net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar'
36
+ require 'io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar'
37
+ require 'commons-codec/commons-codec/1.11/commons-codec-1.11.jar'
38
+ require 'com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar'
39
+ require 'com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar'
5
40
  require 'com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar'
6
- require 'com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar'
7
- require 'net/minidev/json-smart/2.3/json-smart-2.3.jar'
8
- require 'com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar'
9
- require 'org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar'
10
- require 'com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar'
11
- require 'commons-codec/commons-codec/1.14/commons-codec-1.14.jar'
12
- require 'com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar'
13
- require 'org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar'
14
- require 'com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar'
15
- require 'com/google/code/gson/gson/2.8.0/gson-2.8.0.jar'
16
- require 'javax/activation/activation/1.1/activation-1.1.jar'
17
- require 'net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar'
18
- require 'com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar'
19
- require 'com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar'
20
- require 'org/ow2/asm/asm/5.0.4/asm-5.0.4.jar'
21
- require 'com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar'
22
- require 'com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar'
41
+ require 'com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar'
42
+ require 'io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar'
43
+ require 'io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar'
44
+ require 'com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar'
45
+ require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
46
+ require 'org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar'
47
+ require 'org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar'
48
+ require 'com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar'
49
+ require 'io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar'
50
+ require 'com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar'
51
+ require 'org/ow2/asm/asm/9.1/asm-9.1.jar'
52
+ require 'com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar'
53
+ require 'com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar'
54
+ require 'org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar'
55
+ require 'org/json/json/20201115/json-20201115.jar'
56
+ require 'io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar'
57
+ require 'io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar'
58
+ require 'net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar'
59
+ require 'io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar'
60
+ require 'io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar'
61
+ require 'jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar'
62
+ require 'io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar'
63
+ require 'com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar'
64
+ require 'com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar'
65
+ require 'io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar'
66
+ require 'com/nimbusds/content-type/2.1/content-type-2.1.jar'
23
67
  require 'org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar'
24
- require 'org/apache/httpcomponents/httpcore/4.4.11/httpcore-4.4.11.jar'
68
+ require 'com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar'
25
69
  require 'com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar'
26
- require 'com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar'
27
- require 'com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar'
28
- require 'com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar'
29
- require 'com/google/guava/guava/20.0/guava-20.0.jar'
30
- require 'commons-logging/commons-logging/1.2/commons-logging-1.2.jar'
31
- require 'org/json/json/20190722/json-20190722.jar'
32
- require 'org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar'
70
+ require 'com/azure/azure-core/1.21.0/azure-core-1.21.0.jar'
71
+ require 'com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar'
72
+ require 'org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar'
73
+ require 'io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar'
74
+ require 'org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar'
75
+ require 'io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar'
33
76
  end
34
77
 
35
78
  if defined? Jars
79
+ require_jar 'io.netty', 'netty-resolver-dns-native-macos', 'osx-x86_64', '4.1.68.Final'
80
+ require_jar 'com.google.guava', 'guava', '24.1.1-jre'
81
+ require_jar 'com.microsoft.azure.kusto', 'kusto-data', '3.1.3'
82
+ require_jar 'com.microsoft.azure', 'azure-storage', '8.6.6'
83
+ require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '3.1.3'
84
+ require_jar 'io.projectreactor.netty', 'reactor-netty-http', '1.0.11'
85
+ require_jar 'net.java.dev.jna', 'jna', '5.5.0'
86
+ require_jar 'io.netty', 'netty-handler-proxy', '4.1.68.Final'
87
+ require_jar 'org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'
88
+ require_jar 'io.netty', 'netty-codec-socks', '4.1.68.Final'
89
+ require_jar 'org.apache.commons', 'commons-lang3', '3.11'
90
+ require_jar 'org.reactivestreams', 'reactive-streams', '1.0.3'
91
+ require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.2.4'
92
+ require_jar 'io.github.resilience4j', 'resilience4j-core', '1.7.1'
93
+ require_jar 'io.netty', 'netty-codec-dns', '4.1.68.Final'
94
+ require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.9.3'
95
+ require_jar 'io.netty', 'netty-transport', '4.1.68.Final'
96
+ require_jar 'org.jetbrains', 'annotations', '22.0.0'
97
+ require_jar 'io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'
98
+ require_jar 'io.netty', 'netty-transport-native-epoll', 'linux-x86_64', '4.1.68.Final'
99
+ require_jar 'io.vavr', 'vavr', '0.10.2'
100
+ require_jar 'io.netty', 'netty-resolver-dns', '4.1.68.Final'
101
+ require_jar 'com.azure', 'azure-identity', '1.3.7'
102
+ require_jar 'com.fasterxml.jackson.module', 'jackson-module-jaxb-annotations', '2.12.5'
103
+ require_jar 'org.apache.commons', 'commons-text', '1.9'
104
+ require_jar 'net.java.dev.jna', 'jna-platform', '5.6.0'
105
+ require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.13.0'
106
+ require_jar 'io.netty', 'netty-resolver', '4.1.68.Final'
107
+ require_jar 'jakarta.activation', 'jakarta.activation-api', '1.2.1'
108
+ require_jar 'commons-logging', 'commons-logging', '1.2'
109
+ require_jar 'net.minidev', 'accessors-smart', '2.4.7'
110
+ require_jar 'io.netty', 'netty-codec', '4.1.68.Final'
111
+ require_jar 'commons-codec', 'commons-codec', '1.11'
112
+ require_jar 'com.microsoft.azure', 'msal4j-persistence-extension', '1.1.0'
113
+ require_jar 'com.fasterxml.woodstox', 'woodstox-core', '6.2.4'
36
114
  require_jar 'com.github.stephenc.jcip', 'jcip-annotations', '1.0-1'
37
- require_jar 'com.nimbusds', 'nimbus-jose-jwt', '9.3'
38
- require_jar 'net.minidev', 'json-smart', '2.3'
39
- require_jar 'com.microsoft.azure.kusto', 'kusto-data', '2.1.2'
40
- require_jar 'org.apache.commons', 'commons-lang3', '3.9'
41
- require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.9.10.7'
42
- require_jar 'commons-codec', 'commons-codec', '1.14'
43
- require_jar 'com.microsoft.azure.kusto', 'kusto-ingest', '2.1.2'
44
- require_jar 'org.jetbrains', 'annotations', '17.0.0'
45
- require_jar 'com.microsoft.azure', 'azure-storage', '8.3.0'
46
- require_jar 'com.google.code.gson', 'gson', '2.8.0'
47
- require_jar 'javax.activation', 'activation', '1.1'
48
- require_jar 'net.minidev', 'accessors-smart', '1.2'
49
- require_jar 'com.microsoft.azure', 'adal4j', '1.6.5'
50
- require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.9.10'
51
- require_jar 'org.ow2.asm', 'asm', '5.0.4'
52
- require_jar 'com.sun.mail', 'javax.mail', '1.6.1'
53
- require_jar 'com.nimbusds', 'oauth2-oidc-sdk', '6.5'
115
+ require_jar 'com.fasterxml.jackson.core', 'jackson-databind', '2.13.0'
116
+ require_jar 'io.netty', 'netty-transport-native-kqueue', 'osx-x86_64', '4.1.68.Final'
117
+ require_jar 'io.netty', 'netty-handler', '4.1.68.Final'
118
+ require_jar 'com.fasterxml.jackson.core', 'jackson-annotations', '2.13.0'
119
+ require_jar 'com.google.code.findbugs', 'jsr305', '1.3.9'
120
+ require_jar 'org.checkerframework', 'checker-compat-qual', '2.0.0'
121
+ require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.13'
122
+ require_jar 'com.google.j2objc', 'j2objc-annotations', '1.1'
123
+ require_jar 'io.netty', 'netty-codec-http', '4.1.68.Final'
124
+ require_jar 'com.fasterxml.jackson.dataformat', 'jackson-dataformat-xml', '2.12.5'
125
+ require_jar 'org.ow2.asm', 'asm', '9.1'
126
+ require_jar 'com.azure', 'azure-core-http-netty', '1.11.1'
127
+ require_jar 'com.fasterxml.jackson.datatype', 'jackson-datatype-jsr310', '2.12.5'
128
+ require_jar 'org.codehaus.woodstox', 'stax2-api', '4.2.1'
129
+ require_jar 'org.json', 'json', '20201115'
130
+ require_jar 'io.netty', 'netty-tcnative-boringssl-static', '2.0.43.Final'
131
+ require_jar 'io.netty', 'netty-common', '4.1.68.Final'
132
+ require_jar 'net.minidev', 'json-smart', '2.4.7'
133
+ require_jar 'io.projectreactor', 'reactor-core', '3.4.10'
134
+ require_jar 'io.netty', 'netty-buffer', '4.1.68.Final'
135
+ require_jar 'jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.2'
136
+ require_jar 'io.github.resilience4j', 'resilience4j-retry', '1.7.1'
137
+ require_jar 'com.google.errorprone', 'error_prone_annotations', '2.1.3'
138
+ require_jar 'com.univocity', 'univocity-parsers', '2.9.1'
139
+ require_jar 'io.vavr', 'vavr-match', '0.10.2'
140
+ require_jar 'com.nimbusds', 'content-type', '2.1'
54
141
  require_jar 'org.slf4j', 'slf4j-api', '1.8.0-beta4'
55
- require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.11'
142
+ require_jar 'com.microsoft.azure', 'msal4j', '1.11.0'
56
143
  require_jar 'com.nimbusds', 'lang-tag', '1.5'
57
- require_jar 'com.fasterxml.jackson.core', 'jackson-core', '2.9.4'
58
- require_jar 'com.univocity', 'univocity-parsers', '2.1.1'
59
- require_jar 'com.microsoft.azure', 'azure-keyvault-core', '1.0.0'
60
- require_jar 'com.google.guava', 'guava', '20.0'
61
- require_jar 'commons-logging', 'commons-logging', '1.2'
62
- require_jar 'org.json', 'json', '20190722'
63
- require_jar 'org.apache.httpcomponents', 'httpclient', '4.5.8'
144
+ require_jar 'com.azure', 'azure-core', '1.21.0'
145
+ require_jar 'com.nimbusds', 'oauth2-oidc-sdk', '9.7'
146
+ require_jar 'org.apache.httpcomponents', 'httpcore', '4.4.15'
147
+ require_jar 'io.netty', 'netty-codec-http2', '4.1.68.Final'
148
+ require_jar 'org.slf4j', 'slf4j-simple', '1.8.0-beta4'
149
+ require_jar 'io.projectreactor.netty', 'reactor-netty-core', '1.0.11'
64
150
  end
Binary file
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-kusto' #WATCH OUT: we hardcoded usage of this name in one of the classes.
3
- s.version = '1.0.2'
3
+ s.version = '1.0.5'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Writes events to Azure Data Explorer (Kusto)'
6
6
  s.description = 'This is a logstash output plugin used to write events to an Azure Data Explorer (a.k.a Kusto)'
@@ -28,8 +28,9 @@ Gem::Specification.new do |s|
28
28
  s.add_development_dependency 'flores'
29
29
  s.add_development_dependency 'logstash-input-generator'
30
30
  s.add_development_dependency 'ruby-maven', '~> 3.3.11'
31
+ s.add_development_dependency 'rspec_junit_formatter'
31
32
 
32
33
  # Jar dependencies
33
- s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2"
34
+ s.requirements << "jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'"
34
35
  s.add_runtime_dependency 'jar-dependencies'
35
36
  end
@@ -5,12 +5,15 @@ require 'logstash/outputs/kusto/ingestor'
5
5
 
6
6
  describe LogStash::Outputs::Kusto::Ingestor do
7
7
 
8
- let(:ingest_url) { "mycluster" }
8
+ let(:ingest_url) { "https://ingest-sdkse2etest.eastus.kusto.windows.net/" }
9
9
  let(:app_id) { "myid" }
10
10
  let(:app_key) { LogStash::Util::Password.new("mykey") }
11
11
  let(:app_tenant) { "mytenant" }
12
12
  let(:database) { "mydatabase" }
13
13
  let(:table) { "mytable" }
14
+ let(:proxy_host) { "localhost" }
15
+ let(:proxy_port) { 80 }
16
+ let(:proxy_protocol) { "http" }
14
17
  let(:json_mapping) { "mymapping" }
15
18
  let(:delete_local) { false }
16
19
  let(:logger) { spy('logger') }
@@ -21,7 +24,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
21
24
  # note that this will cause an internal error since connection is being tried.
22
25
  # however we still want to test that all the java stuff is working as expected
23
26
  expect {
24
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, logger)
27
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
25
28
  ingestor.stop
26
29
  }.not_to raise_error
27
30
  end
@@ -32,7 +35,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
32
35
  dynamic_name_array.each do |test_database|
33
36
  it "with database: #{test_database}" do
34
37
  expect {
35
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, logger)
38
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
36
39
  ingestor.stop
37
40
  }.to raise_error(LogStash::ConfigurationError)
38
41
  end
@@ -43,7 +46,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
43
46
  dynamic_name_array.each do |test_table|
44
47
  it "with database: #{test_table}" do
45
48
  expect {
46
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, logger)
49
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
47
50
  ingestor.stop
48
51
  }.to raise_error(LogStash::ConfigurationError)
49
52
  end
@@ -51,16 +54,25 @@ describe LogStash::Outputs::Kusto::Ingestor do
51
54
  end
52
55
 
53
56
  context 'doesnt allow mapping to have some dynamic part' do
54
- dynamic_name_array.each do |test_json_mapping|
55
- it "with database: #{test_json_mapping}" do
57
+ dynamic_name_array.each do |json_mapping|
58
+ it "with database: #{json_mapping}" do
56
59
  expect {
57
- ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, test_json_mapping, delete_local, logger)
60
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
58
61
  ingestor.stop
59
62
  }.to raise_error(LogStash::ConfigurationError)
60
63
  end
61
64
  end
62
65
  end
63
66
 
67
+ context 'proxy protocol has to be http or https' do
68
+ it "with proxy protocol: socks" do
69
+ expect {
70
+ ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
71
+ ingestor.stop
72
+ }.to raise_error(LogStash::ConfigurationError)
73
+ end
74
+ end
75
+
64
76
  end
65
77
 
66
78
  # describe 'receiving events' do
@@ -1,5 +1,4 @@
1
1
  # encoding: utf-8
2
- require 'logstash/devutils/rspec/spec_helper'
3
2
  require 'logstash/outputs/kusto'
4
3
  require 'logstash/codecs/plain'
5
4
  require 'logstash/event'
@@ -7,13 +6,16 @@ require 'logstash/event'
7
6
  describe LogStash::Outputs::Kusto do
8
7
 
9
8
  let(:options) { { "path" => "./kusto_tst/%{+YYYY-MM-dd-HH-mm}",
10
- "ingest_url" => "mycluster",
9
+ "ingest_url" => "https://ingest-sdkse2etest.eastus.kusto.windows.net/",
11
10
  "app_id" => "myid",
12
11
  "app_key" => "mykey",
13
- "app_tenant" => "mytenant",
12
+ "app_tenant" => "mytenant",
14
13
  "database" => "mydatabase",
15
14
  "table" => "mytable",
16
- "json_mapping" => "mymapping"
15
+ "json_mapping" => "mymapping",
16
+ "proxy_host" => "localhost",
17
+ "proxy_port" => 3128,
18
+ "proxy_protocol" => "https"
17
19
  } }
18
20
 
19
21
  describe '#register' do
@@ -50,5 +52,5 @@ describe LogStash::Outputs::Kusto do
50
52
  end
51
53
 
52
54
  end
53
-
55
+
54
56
  end
data/spec/spec_helpers.rb CHANGED
@@ -18,4 +18,4 @@ RSpec.configure do |config|
18
18
  $stdout = STDOUT
19
19
  $stderr = STDERR
20
20
  end
21
- end
21
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-kusto
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.0.5
5
5
  platform: java
6
6
  authors:
7
7
  - Tamir Kamara
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-12-27 00:00:00.000000000 Z
12
+ date: 2022-08-04 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -18,8 +18,8 @@ dependencies:
18
18
  - !ruby/object:Gem::Version
19
19
  version: '2.0'
20
20
  name: logstash-core-plugin-api
21
- type: :runtime
22
21
  prerelease: false
22
+ type: :runtime
23
23
  version_requirements: !ruby/object:Gem::Requirement
24
24
  requirements:
25
25
  - - "~>"
@@ -32,8 +32,8 @@ dependencies:
32
32
  - !ruby/object:Gem::Version
33
33
  version: '0'
34
34
  name: logstash-codec-json_lines
35
- type: :runtime
36
35
  prerelease: false
36
+ type: :runtime
37
37
  version_requirements: !ruby/object:Gem::Requirement
38
38
  requirements:
39
39
  - - ">="
@@ -46,8 +46,8 @@ dependencies:
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  name: logstash-codec-line
49
- type: :runtime
50
49
  prerelease: false
50
+ type: :runtime
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
53
  - - ">="
@@ -60,8 +60,8 @@ dependencies:
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  name: logstash-devutils
63
- type: :development
64
63
  prerelease: false
64
+ type: :development
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
67
  - - ">="
@@ -74,8 +74,8 @@ dependencies:
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  name: flores
77
- type: :development
78
77
  prerelease: false
78
+ type: :development
79
79
  version_requirements: !ruby/object:Gem::Requirement
80
80
  requirements:
81
81
  - - ">="
@@ -88,8 +88,8 @@ dependencies:
88
88
  - !ruby/object:Gem::Version
89
89
  version: '0'
90
90
  name: logstash-input-generator
91
- type: :development
92
91
  prerelease: false
92
+ type: :development
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
95
  - - ">="
@@ -102,13 +102,27 @@ dependencies:
102
102
  - !ruby/object:Gem::Version
103
103
  version: 3.3.11
104
104
  name: ruby-maven
105
- type: :development
106
105
  prerelease: false
106
+ type: :development
107
107
  version_requirements: !ruby/object:Gem::Requirement
108
108
  requirements:
109
109
  - - "~>"
110
110
  - !ruby/object:Gem::Version
111
111
  version: 3.3.11
112
+ - !ruby/object:Gem::Dependency
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ name: rspec_junit_formatter
119
+ prerelease: false
120
+ type: :development
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - ">="
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
112
126
  - !ruby/object:Gem::Dependency
113
127
  requirement: !ruby/object:Gem::Requirement
114
128
  requirements:
@@ -116,8 +130,8 @@ dependencies:
116
130
  - !ruby/object:Gem::Version
117
131
  version: '0'
118
132
  name: jar-dependencies
119
- type: :runtime
120
133
  prerelease: false
134
+ type: :runtime
121
135
  version_requirements: !ruby/object:Gem::Requirement
122
136
  requirements:
123
137
  - - ">="
@@ -135,38 +149,82 @@ files:
135
149
  - Gemfile
136
150
  - LICENSE
137
151
  - README.md
138
- - lib/com/fasterxml/jackson/core/jackson-annotations/2.9.10/jackson-annotations-2.9.10.jar
139
- - lib/com/fasterxml/jackson/core/jackson-core/2.9.4/jackson-core-2.9.4.jar
140
- - lib/com/fasterxml/jackson/core/jackson-databind/2.9.10.7/jackson-databind-2.9.10.7.jar
152
+ - SECURITY.md
153
+ - lib/com/azure/azure-core-http-netty/1.11.1/azure-core-http-netty-1.11.1.jar
154
+ - lib/com/azure/azure-core/1.21.0/azure-core-1.21.0.jar
155
+ - lib/com/azure/azure-identity/1.3.7/azure-identity-1.3.7.jar
156
+ - lib/com/fasterxml/jackson/core/jackson-annotations/2.13.0/jackson-annotations-2.13.0.jar
157
+ - lib/com/fasterxml/jackson/core/jackson-core/2.13.0/jackson-core-2.13.0.jar
158
+ - lib/com/fasterxml/jackson/core/jackson-databind/2.13.0/jackson-databind-2.13.0.jar
159
+ - lib/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/2.12.5/jackson-dataformat-xml-2.12.5.jar
160
+ - lib/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.12.5/jackson-datatype-jsr310-2.12.5.jar
161
+ - lib/com/fasterxml/jackson/module/jackson-module-jaxb-annotations/2.12.5/jackson-module-jaxb-annotations-2.12.5.jar
162
+ - lib/com/fasterxml/woodstox/woodstox-core/6.2.4/woodstox-core-6.2.4.jar
141
163
  - lib/com/github/stephenc/jcip/jcip-annotations/1.0-1/jcip-annotations-1.0-1.jar
142
- - lib/com/google/code/gson/gson/2.8.0/gson-2.8.0.jar
143
- - lib/com/google/guava/guava/20.0/guava-20.0.jar
144
- - lib/com/microsoft/azure/adal4j/1.6.5/adal4j-1.6.5.jar
145
- - lib/com/microsoft/azure/azure-keyvault-core/1.0.0/azure-keyvault-core-1.0.0.jar
146
- - lib/com/microsoft/azure/azure-storage/8.3.0/azure-storage-8.3.0.jar
147
- - lib/com/microsoft/azure/kusto/kusto-data/2.1.2/kusto-data-2.1.2.jar
148
- - lib/com/microsoft/azure/kusto/kusto-ingest/2.1.2/kusto-ingest-2.1.2.jar
164
+ - lib/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar
165
+ - lib/com/google/errorprone/error_prone_annotations/2.1.3/error_prone_annotations-2.1.3.jar
166
+ - lib/com/google/guava/guava/24.1.1-jre/guava-24.1.1-jre.jar
167
+ - lib/com/google/j2objc/j2objc-annotations/1.1/j2objc-annotations-1.1.jar
168
+ - lib/com/microsoft/azure/azure-keyvault-core/1.2.4/azure-keyvault-core-1.2.4.jar
169
+ - lib/com/microsoft/azure/azure-storage/8.6.6/azure-storage-8.6.6.jar
170
+ - lib/com/microsoft/azure/kusto/kusto-data/3.1.3/kusto-data-3.1.3.jar
171
+ - lib/com/microsoft/azure/kusto/kusto-ingest/3.1.3/kusto-ingest-3.1.3.jar
172
+ - lib/com/microsoft/azure/msal4j-persistence-extension/1.1.0/msal4j-persistence-extension-1.1.0.jar
173
+ - lib/com/microsoft/azure/msal4j/1.11.0/msal4j-1.11.0.jar
174
+ - lib/com/nimbusds/content-type/2.1/content-type-2.1.jar
149
175
  - lib/com/nimbusds/lang-tag/1.5/lang-tag-1.5.jar
150
- - lib/com/nimbusds/nimbus-jose-jwt/9.3/nimbus-jose-jwt-9.3.jar
151
- - lib/com/nimbusds/oauth2-oidc-sdk/6.5/oauth2-oidc-sdk-6.5.jar
152
- - lib/com/sun/mail/javax.mail/1.6.1/javax.mail-1.6.1.jar
153
- - lib/com/univocity/univocity-parsers/2.1.1/univocity-parsers-2.1.1.jar
154
- - lib/commons-codec/commons-codec/1.14/commons-codec-1.14.jar
176
+ - lib/com/nimbusds/nimbus-jose-jwt/9.9.3/nimbus-jose-jwt-9.9.3.jar
177
+ - lib/com/nimbusds/oauth2-oidc-sdk/9.7/oauth2-oidc-sdk-9.7.jar
178
+ - lib/com/univocity/univocity-parsers/2.9.1/univocity-parsers-2.9.1.jar
179
+ - lib/commons-codec/commons-codec/1.11/commons-codec-1.11.jar
155
180
  - lib/commons-logging/commons-logging/1.2/commons-logging-1.2.jar
156
- - lib/javax/activation/activation/1.1/activation-1.1.jar
181
+ - lib/io/github/resilience4j/resilience4j-core/1.7.1/resilience4j-core-1.7.1.jar
182
+ - lib/io/github/resilience4j/resilience4j-retry/1.7.1/resilience4j-retry-1.7.1.jar
183
+ - lib/io/netty/netty-buffer/4.1.68.Final/netty-buffer-4.1.68.Final.jar
184
+ - lib/io/netty/netty-codec-dns/4.1.68.Final/netty-codec-dns-4.1.68.Final.jar
185
+ - lib/io/netty/netty-codec-http/4.1.68.Final/netty-codec-http-4.1.68.Final.jar
186
+ - lib/io/netty/netty-codec-http2/4.1.68.Final/netty-codec-http2-4.1.68.Final.jar
187
+ - lib/io/netty/netty-codec-socks/4.1.68.Final/netty-codec-socks-4.1.68.Final.jar
188
+ - lib/io/netty/netty-codec/4.1.68.Final/netty-codec-4.1.68.Final.jar
189
+ - lib/io/netty/netty-common/4.1.68.Final/netty-common-4.1.68.Final.jar
190
+ - lib/io/netty/netty-handler-proxy/4.1.68.Final/netty-handler-proxy-4.1.68.Final.jar
191
+ - lib/io/netty/netty-handler/4.1.68.Final/netty-handler-4.1.68.Final.jar
192
+ - lib/io/netty/netty-resolver-dns-native-macos/4.1.68.Final/netty-resolver-dns-native-macos-4.1.68.Final-osx-x86_64.jar
193
+ - lib/io/netty/netty-resolver-dns/4.1.68.Final/netty-resolver-dns-4.1.68.Final.jar
194
+ - lib/io/netty/netty-resolver/4.1.68.Final/netty-resolver-4.1.68.Final.jar
195
+ - lib/io/netty/netty-tcnative-boringssl-static/2.0.43.Final/netty-tcnative-boringssl-static-2.0.43.Final.jar
196
+ - lib/io/netty/netty-transport-native-epoll/4.1.68.Final/netty-transport-native-epoll-4.1.68.Final-linux-x86_64.jar
197
+ - lib/io/netty/netty-transport-native-kqueue/4.1.68.Final/netty-transport-native-kqueue-4.1.68.Final-osx-x86_64.jar
198
+ - lib/io/netty/netty-transport-native-unix-common/4.1.68.Final/netty-transport-native-unix-common-4.1.68.Final.jar
199
+ - lib/io/netty/netty-transport/4.1.68.Final/netty-transport-4.1.68.Final.jar
200
+ - lib/io/projectreactor/netty/reactor-netty-core/1.0.11/reactor-netty-core-1.0.11.jar
201
+ - lib/io/projectreactor/netty/reactor-netty-http/1.0.11/reactor-netty-http-1.0.11.jar
202
+ - lib/io/projectreactor/reactor-core/3.4.10/reactor-core-3.4.10.jar
203
+ - lib/io/vavr/vavr-match/0.10.2/vavr-match-0.10.2.jar
204
+ - lib/io/vavr/vavr/0.10.2/vavr-0.10.2.jar
205
+ - lib/jakarta/activation/jakarta.activation-api/1.2.1/jakarta.activation-api-1.2.1.jar
206
+ - lib/jakarta/xml/bind/jakarta.xml.bind-api/2.3.2/jakarta.xml.bind-api-2.3.2.jar
157
207
  - lib/logstash-output-kusto_jars.rb
158
208
  - lib/logstash/outputs/kusto.rb
159
209
  - lib/logstash/outputs/kusto/ingestor.rb
160
210
  - lib/logstash/outputs/kusto/interval.rb
161
- - lib/net/minidev/accessors-smart/1.2/accessors-smart-1.2.jar
162
- - lib/net/minidev/json-smart/2.3/json-smart-2.3.jar
163
- - lib/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar
164
- - lib/org/apache/httpcomponents/httpclient/4.5.8/httpclient-4.5.8.jar
165
- - lib/org/apache/httpcomponents/httpcore/4.4.11/httpcore-4.4.11.jar
166
- - lib/org/jetbrains/annotations/17.0.0/annotations-17.0.0.jar
167
- - lib/org/json/json/20190722/json-20190722.jar
168
- - lib/org/ow2/asm/asm/5.0.4/asm-5.0.4.jar
211
+ - lib/net/java/dev/jna/jna-platform/5.6.0/jna-platform-5.6.0.jar
212
+ - lib/net/java/dev/jna/jna/5.5.0/jna-5.5.0.jar
213
+ - lib/net/minidev/accessors-smart/2.4.7/accessors-smart-2.4.7.jar
214
+ - lib/net/minidev/json-smart/2.4.7/json-smart-2.4.7.jar
215
+ - lib/org/apache/commons/commons-lang3/3.11/commons-lang3-3.11.jar
216
+ - lib/org/apache/commons/commons-text/1.9/commons-text-1.9.jar
217
+ - lib/org/apache/httpcomponents/httpclient/4.5.13/httpclient-4.5.13.jar
218
+ - lib/org/apache/httpcomponents/httpcore/4.4.15/httpcore-4.4.15.jar
219
+ - lib/org/checkerframework/checker-compat-qual/2.0.0/checker-compat-qual-2.0.0.jar
220
+ - lib/org/codehaus/mojo/animal-sniffer-annotations/1.14/animal-sniffer-annotations-1.14.jar
221
+ - lib/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar
222
+ - lib/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar
223
+ - lib/org/json/json/20201115/json-20201115.jar
224
+ - lib/org/ow2/asm/asm/9.1/asm-9.1.jar
225
+ - lib/org/reactivestreams/reactive-streams/1.0.3/reactive-streams-1.0.3.jar
169
226
  - lib/org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar
227
+ - lib/org/slf4j/slf4j-simple/1.8.0-beta4/slf4j-simple-1.8.0-beta4.jar
170
228
  - logstash-output-kusto.gemspec
171
229
  - spec/outputs/kusto/ingestor_spec.rb
172
230
  - spec/outputs/kusto_spec.rb
@@ -192,8 +250,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
192
250
  - !ruby/object:Gem::Version
193
251
  version: '0'
194
252
  requirements:
195
- - jar 'com.microsoft.azure.kusto, kusto-ingest, 2.1.2
196
- rubygems_version: 3.0.6
253
+ - jar 'com.microsoft.azure.kusto, kusto-ingest, 3.1.3'
254
+ rubygems_version: 3.2.29
197
255
  signing_key:
198
256
  specification_version: 4
199
257
  summary: Writes events to Azure Data Explorer (Kusto)