fluent-plugin-kafka-xst 0.19.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a6c24e3153f666f2eef5760a5c2596a41b43ec38
+  data.tar.gz: 074d9ba7f2f515cf6f09e58fac80075ff738ea30
+SHA512:
+  metadata.gz: 176d69e9c6dc1eb75c1ecdfe5104b4cbc46b5551b9053041e8aa85f3d52dc10a958fde5a8df41112cce2e3cd8bc003a9644e66a557af4d40b21102950751725e
+  data.tar.gz: c047a78bb81e8edd5dbbcde4d737752815a99448ef7020fc0eb05d57a6cd6adf9262a0d18b63e93e7e00f4533045519b4884f38a3939d0eee373a152f235b491

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,72 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-kafka/#faq) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-kafka version:
+        - ruby-kafka version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,39 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: "waiting-for-triage"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-kafka/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

data/.github/workflows/linux.yml

@@ -0,0 +1,45 @@
+name: linux
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    env:
+      USE_RDKAFKA: 1
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+        os:
+          - ubuntu-latest
+        rdkafka_versions:
+          - { min: '>= 0.6.0', max: '< 0.12.0' }
+          - { min: '>= 0.12.0', max: '>= 0.12.0' }
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
+    steps:
+    - uses: actions/checkout@v3
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: Install confluent-kafka
+      run: |
+        sudo apt install -V -y gnupg2 wget
+        wget https://packages.confluent.io/deb/6.0/archive.key
+        sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
+        sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
+        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
+        sudo apt update
+        sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
+    - name: unit testing
+      env:
+        CI: true
+        RDKAFKA_VERSION_MIN_RANGE: ${{ matrix.rdkafka_versions.min }}
+        RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
+      run: |
+        sudo ./ci/prepare-kafka-server.sh
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,24 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v8
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,enhancement,help wanted,waiting-for-triage"
+        exempt-pr-labels: "bug,enhancement,help wanted,waiting-for-triage"
+        exempt-all-assignees: true
+        exempt-all-milestones: true

data/.gitignore

@@ -0,0 +1,2 @@
+/Gemfile.lock
+*.swp

data/ChangeLog

@@ -0,0 +1,344 @@
+Release 0.19.0 - 2023/04/26
+	* out_kafka2: Add support for AWS IAM authentication
+	* in_kafka, in_kafka_group, out_kafka2: Add support for ssl client cert key password
+	* out_rdkafka2: Mask `ssl_client_cert_key_password` on dumping it to log
+	* out_rdkafka2: Support rdkafka-ruby 0.12
+
+Release 0.18.1 - 2022/08/17
+	* out_kafka2: Fix a bug that it doesn't respect `chunk_limit_records` and `chunk_limit_size`
+
+Release 0.18.0 - 2022/07/21
+	* out_kafka2: Keep alive Kafka connections between flushes
+	* out_rdkafka2: Enable to set SASL credentials via `username` and `password` parameters
+	* out_kafka2/out_rdkafka2: Add `record_key` parameter
+
+Release 0.17.5 - 2022/03/18
+	* out_kafka2: Add `resolve_seed_brokers` parameter
+
+Release 0.17.4 - 2022/01/25
+	* in_kafka_group: Add `refresh_topic_interval` parameter
+
+Release 0.17.3 - 2021/11/26
+	* output: Suppress large warning logs for events skipped by `max_send_limit_bytes`
+
+Release 0.17.2 - 2021/10/14
+	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
+	* out_rdkafka2: Support `use_event_time` parameter
+	* out_rdkafka2: Fix a potential bug that the plugin might exit without receiving responses from Kafka.
+
+Release 0.17.1 - 2021/09/24
+	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
+	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter
+	* out_kafka2.rb: Fix one more Ruby 3.0 keyword arguments issue
+
+Release 0.17.0 - 2021/08/30
+	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
+	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs
+	* out_kafka/out_kafka2: fix description for `exclude_message_key` option
+
+Release 0.16.3 - 2021/05/17
+	* in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
+
+Release 0.16.2 - 2021/05/17
+	* in_kafka, in_kafka_group: Support Ruby 3.0 keyword arguments interop
+
+Release 0.16.1 - 2021/04/14
+	* out_kafka/out_kafka_buffered: Support Ruby 3.0.0 keyword arguments interop
+	* kafka_plugin_util: Treat empty string in read_ssl_file as nil
+
+Release 0.16.0 - 2021/01/25
+
+	* input: Add `tag_source` and `record_tag_key` parameters for using record field as tag
+	* in_kafka_group: Use NumericParser for floating point
+
+Release 0.15.3 - 2020/12/08
+
+	* in_kafka: Fix `record_time_key` parameter not working
+
+Release 0.15.2 - 2020/09/30
+
+	* input: Support 3rd party parser
+
+Release 0.15.1 - 2020/09/17
+
+	* out_kafka2: Fix wrong class name for configuration error
+
+Release 0.15.0 - 2020/09/14
+
+	* Add experimental `in_rdkafka_group`
+	* in_kafka: Expose `ssl_verify_hostname` parameter
+
+Release 0.14.2 - 2020/08/26
+
+	* in_kafka_group: Add `add_headers` parameter
+	* out_kafka2/out_rdkafka2: Support `discard_kafka_delivery_failed` parameter
+
+Release 0.14.1 - 2020/08/11
+
+	* kafka_producer_ext: Fix regression by v0.14.0 changes
+
+Release 0.14.0 - 2020/08/07
+
+	* Update ruby-kafka dependency to v1.2.0 or later. Check https://github.com/zendesk/ruby-kafka#compatibility
+	* kafka_producer_ext: Follow Paritioner API change
+
+Release 0.13.1 - 2020/07/17
+
+	* in_kafka_group: Support ssl_verify_hostname parameter
+	* in_kafka_group: Support regex based topics
+	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
+
+Release 0.13.0 - 2020/03/09
+
+	* Accept ruby-kafka v1 or later
+
+Release 0.12.4 - 2020/03/03
+
+	* output: Follow rdkafka log level
+
+Release 0.12.3 - 2020/02/06
+
+	* output: Show warning message for v0.12 plugins
+
+Release 0.12.2 - 2020/01/07
+
+	* input: Refer sasl_over_ssl parameter in plain SASL
+
+Release 0.12.1 - 2019/10/14
+
+	* input: Add time_source parameter to replace use_record_time
+
+Release 0.12.0 - 2019/09/27
+
+	* Support rdkafka-ruby v0.7. Drop v0.6 support.
+
+Release 0.11.2 - 2019/09/27
+
+	* out_rdkafka2: Add share_producer parameter
+
+Release 0.11.1 - 2019/08/22
+
+	* out_kafka2: Fix mutating issue of headers
+
+Release 0.11.0 - 2019/07/17
+
+	* out_kafka2: Add connect_timeout / socket_timeout parameters
+	* out_kafka2/out_rdkafka2: Add headers / headers_from_record parameters
+	* output: Add use_event_time parameter to use fluentd event time for CreateTime
+	* input: Add get_kafka_client_log parameter. Default is `false`
+
+Release 0.10.0 - 2019/07/03
+
+	* output: Fixed max_send_limit_bytes parameter to be in all kafka outputs
+	* output: Add ssl_verify_hostname parameter
+	* rdkafka output: Fix exceptiona handling
+	* Update ruby-kafka version to v0.7.8 or later
+
+Release 0.9.6 - 2019/05/28
+
+	* out_kafka2: Add metadata nil check
+
+Release 0.9.5 - 2019/05/23
+
+	* output: Support sasl_over_ssl parameter to not use SSL with SASL/GSSAPI
+
+Release 0.9.4 - 2019/05/13
+
+	* out_rdkafka2: Add metadata nil check
+
+Release 0.9.3 - 2019/05/04
+
+	* in_kafka_group: Add connect_timeout and socket_timeout parameters
+
+Release 0.9.2 - 2019/03/26
+
+	* out_kafka_buffered: Fix typo of partition_key usage
+
+Release 0.9.1 - 2019/03/25
+
+	* output: Support sasl_over_ssl parameter
+	* Support ruby-kafka 0.7.6
+
+Release 0.9.0 - 2019/02/22
+
+	* Add v1 API based rdkafka2 output plugin
+	* out_kafka2: Add use_default_for_unknown_topic parameter
+
+Release 0.8.4 - 2019/01/18
+
+	* in_kafka_group: Support integer/float time field with time_format
+
+Release 0.8.3 - 2018/12/16
+
+	* output: Add ssl_client_cert_chain parameter
+
+Release 0.8.2 - 2018/12/07
+
+	* output: Add idempotent parameter
+	* out_kafka2: Fix DeliveryFailed signature mismatch
+
+Release 0.8.1 - 2018/11/20
+
+	* input: Support SASL without SSL
+
+Release 0.8.0 - 2018/10/18
+
+	* output: Support SASL without SSL
+	* output: Add rdkafka based output
+	* Update ruby-kafka dependency to v0.7 or later
+
+Release 0.7.9 - 2018/09/11
+
+	* in_kafka_group: Add fetcher_max_queue_size parameter
+
+Release 0.7.8 - 2018/08/29
+
+	* input: Fix regression of kafka initialization
+
+Release 0.7.7 - 2018/08/27
+
+	* in_kafka_group: Add retry_limit and disable_retry_limit parameters
+	* input: Support SSL parameters
+	* Add ssl_ca_certs_from_system parameter
+	* Make password parameter secret
+
+Release 0.7.6 - 2018/08/15
+
+	* out_kafka2: Add ignore_exceptions and exception_backup parameters
+
+Release 0.7.5 - 2018/08/14
+
+	* out_kafka2: Fix `message_key_key` parameter handling
+	* Limit ruby-kafka version('< 0.7.0') to avoid runtime error
+
+Release 0.7.4 - 2018/07/11
+
+	* out_kafka2: Need event_emitter helper for notification
+
+Release 0.7.3 - 2018/05/30
+
+	* output: Use SSL endpoint when ssl_client_cert is true
+
+Release 0.7.2 - 2018/03/12
+
+	* output: Fix the regression of get_kafka_client_log parameter
+
+Release 0.7.1 - 2018/03/05
+
+	* out_kafka2: Fix the regression of kafka client initialization
+
+Release 0.7.0 - 2018/02/27
+
+	* output: SASL PLAIN/SCRAM support
+	* input: Fix TimeParser location bug in v1
+	* out_kafka2: Fix warning condition for tag chunk key
+
+Release 0.6.6 - 2017/12/25
+
+	* output: Make topic/partition/partition_key/message_key configurable
+
+Release 0.6.5 - 2017/12/14
+
+	* in_kafka_group: Add client_id parameter
+
+Release 0.6.4 - 2017/11/23
+
+	* Multi worker support for v0.14
+	* Add kafka_message_key parameter to input plugins
+	* Relax ruby-kafka version for 0.5 or later
+	* Use oj for json when it is installed
+
+Release 0.6.3 - 2017/11/14
+
+	* in_kafka_group: re-create consumer when error happens during event fetch
+
+Release 0.6.2 - 2017/11/1
+
+	* Fix ltsv parsing issue which generates symbol keys
+
+Release 0.6.1 - 2017/08/30
+
+	* Add stats and datadog monitoring support
+	* ssl_ca_certs now accepts multiple paths
+	* Fix bug by ruby-kafka 0.4.1 changes
+	* Update ruby-kafka dependency to v0.4.1
+
+Release 0.6.0 - 2017/07/25
+
+	* Add principal and keytab parameters for SASL support
+
+Release 0.5.7 - 2017/07/13
+
+	* out_kafka_buffered: Add kafka_agg_max_messages parameter
+
+Release 0.5.6 - 2017/07/10
+
+	* output: Add ActiveSupport notification support
+
+Release 0.5.5 - 2017/04/19
+
+	* output: Some trace log level changed to debug
+	* out_kafka_buffered: Add discard_kafka_delivery_failed parameter
+
+Release 0.5.4 - 2017/04/12
+
+	* out_kafka_buffered: Add max_send_limit_bytes parameter
+	* out_kafka: Improve buffer overflow handling of ruby-kafka
+
+Release 0.5.3 - 2017/02/13
+
+	* Relax ruby-kafka dependency
+
+Release 0.5.2 - 2017/02/13
+
+	* in_kafka_group: Add max_bytes parameter
+
+Release 0.5.1 - 2017/02/06
+
+	* in_kafka_group: Fix uninitialized constant error
+
+Release 0.5.0 - 2017/01/17
+
+	* output: Add out_kafka2 plugin with v0.14 API
+
+Release 0.4.2 - 2016/12/10
+
+	* input: Add use_record_time and time_format parameters
+	* Update ruby-kafka dependency to 0.3.16.beta2
+
+Release 0.4.1 - 2016/12/01
+
+	* output: Support specifying partition
+
+Release 0.4.0 - 2016/11/08
+
+	* Remove zookeeper dependency
+
+Release 0.3.5 - 2016/10/21
+
+	* output: Support message key and related parameters. #91
+
+Release 0.3.4 - 2016/10/20
+
+	* output: Add exclude_topic_key and exclude_partition_key. #89
+
+Release 0.3.3 - 2016/10/17
+
+	* out_kafka_buffered: Add get_kafka_client_log parameter. #83
+	* out_kafka_buffered: Skip and log invalid record to avoid buffer stuck. #86
+	* in_kafka_group: Add retry_emit_limit to handle BufferQueueLimitError. #87
+
+Release 0.3.2 - 2016/10/06
+
+	* in_kafka_group: Re-fetch events after consumer error. #79
+
+Release 0.3.1 - 2016/08/28
+
+	* output: Change default required_acks to -1. #70
+	* Support ruby version changed to 2.1.0 or later
+
+Release 0.3.0 - 2016/08/24
+
+	* Fully replace poseidon ruby library with ruby-kafka to support latest kafka versions
+
+See git commits for older changes

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafka.gemspec
+gemspec
+
+gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA']

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2014 htgc
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+