fluent-plugin-gcloud-pubsub-custom-compress-batches 1.3.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1b9c7b3dc34f8091993d291f47ab10b9cd20aec0a903bab3d1a3b749871d16b7
+  data.tar.gz: e4e208aa00f236e3c579a281848ed937e482b6240a357defd926affb85ab822c
+SHA512:
+  metadata.gz: b87a13aa2d02eaae3aa1d27dcaeba521a477c0ce3bb1b2890b4d27b678eeb51647df25c6f3bf536681ad6ba331404c06f1c560ae1427990806215ed40adc72ad
+  data.tar.gz: af0a44c4bcaf565c23fed7970d79e3f25428d34e0764a4481a62f03e58c67c9f462d8e1bb50fe3bc5ab64e8e610db4305e4f7927dc15d945166d6a5c6cbc3c47

data/.gitignore

@@ -0,0 +1,11 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version

data/.rubocop.yml

@@ -0,0 +1,46 @@
+---
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  # Matches the minimum version in .travis.yml
+  TargetRubyVersion: 2.4
+
+Style/StringLiterals:
+  EnforcedStyle: "double_quotes"
+
+# New cops: https://docs.rubocop.org/en/latest/versioning/
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: true
+
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+
+Lint/RaiseException:
+  Enabled: true
+
+Lint/StructNewOverride:
+  Enabled: true
+
+Style/ExponentialNotation:
+  Enabled: true
+
+Style/HashEachMethods:
+  Enabled: true
+
+Style/HashTransformKeys:
+  Enabled: true
+
+Style/HashTransformValues:
+  Enabled: true
+
+Style/SlicingWithRange:
+  Enabled: true
+
+Style/TrailingCommaInHashLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInArrayLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInArguments:
+  EnforcedStyleForMultiline: comma

data/.rubocop_todo.yml

@@ -0,0 +1,98 @@
+---
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2020-05-12 17:34:19 +0100 using RuboCop version 0.83.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 3
+# Configuration parameters: IgnoredMethods.
+Metrics/AbcSize:
+  Max: 50
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 30
+
+# Offense count: 1
+# Configuration parameters: IgnoredPatterns.
+# SupportedStyles: snake_case, camelCase
+Naming/MethodName:
+  EnforcedStyle: snake_case
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, EnforcedStyle.
+# SupportedStyles: nested, compact
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'lib/fluent/plugin/in_gcloud_pubsub.rb'
+    - 'lib/fluent/plugin/out_gcloud_pubsub.rb'
+    - 'test/test_helper.rb'
+
+# Offense count: 6
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/fluent/plugin/gcloud_pubsub/client.rb'
+    - 'lib/fluent/plugin/in_gcloud_pubsub.rb'
+    - 'lib/fluent/plugin/out_gcloud_pubsub.rb'
+
+# Offense count: 1
+# Configuration parameters: AllowedVariables.
+Style/GlobalVars:
+  Exclude:
+    - 'test/test_helper.rb'
+
+# Offense count: 1
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Exclude:
+    - 'lib/fluent/plugin/gcloud_pubsub/client.rb'
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/IfUnlessModifier:
+  Exclude:
+    - 'lib/fluent/plugin/gcloud_pubsub/client.rb'
+
+# Offense count: 1
+Style/MethodMissingSuper:
+  Exclude:
+    - 'test/test_helper.rb'
+
+# Offense count: 1
+Style/MissingRespondToMissing:
+  Exclude:
+    - 'test/test_helper.rb'
+
+# Offense count: 260
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
+# SupportedStyles: single_quotes, double_quotes
+Style/StringLiterals:
+  Exclude:
+    - 'Gemfile'
+    - 'Rakefile'
+    - 'fluent-plugin-gcloud-pubsub-custom.gemspec'
+    - 'lib/fluent/plugin/gcloud_pubsub/client.rb'
+    - 'lib/fluent/plugin/in_gcloud_pubsub.rb'
+    - 'lib/fluent/plugin/out_gcloud_pubsub.rb'
+    - 'test/plugin/test_in_gcloud_pubsub.rb'
+    - 'test/plugin/test_out_gcloud_pubsub.rb'
+    - 'test/test_helper.rb'
+
+# Offense count: 36
+# Cop supports --auto-correct.
+# Configuration parameters: AutoCorrect, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 120

data/.travis.yml

@@ -0,0 +1,26 @@
+language: ruby
+
+rvm:
+  - 2.5
+  - 2.6
+  - 2.7
+  - ruby-head
+
+gemfile:
+  - Gemfile
+
+branches:
+  only:
+    - master
+    - gocardless
+
+before_install: gem update bundler
+script:
+  - bundle exec rake test
+  - bundle exec rubocop
+
+sudo: false
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head

data/CHANGELOG.md

@@ -0,0 +1,143 @@
+## ChangeLog
+
+### Release 1.3.3 - 2021/01/05
+
+- Add support for payload compression with `compress_batches`
+
+### Release 1.3.2 - 2019/08/16
+
+- Input plugin
+  - Make `topic` parameter optional
+
+### Release 1.3.1 - 2018/06/08
+
+- Input plugin
+  - Dumps raw string to logs for parsing errors
+
+### Release 1.3.0 - 2018/05/18
+
+- Make attributes available 
+
+### Release 1.2.0 - 2018/04/24
+
+- Output plugin
+  - Support inject section 
+
+### Release 1.1.0 - 2018/03/30
+
+- Output plugin
+  - The placeholder is now available in topic param
+
+### Release 1.0.3 - 2018/03/29
+
+- Bump up google-cloud-pubsub to v0.30.x
+- Make config file Fluentd v1 compatible
+
+### Release 1.0.2 - 2017/09/11
+
+- Bump up google-cloud-pubsub to v0.27.x
+
+### Release 1.0.1 - 2017/09/05
+
+- Bump up google-cloud-pubsub to v0.26.x
+
+### Release 1.0.0 - 2017/06/23
+
+- Fluentd v0.14 ready
+  - Fluentd v0.12 is not supported in the later version
+
+### Release 0.4.6 - 2017/05/14
+
+- Output plugin
+  - Make messages exceeding configured size not be published because Pub/Sub clients cannot receive it
+
+### Release 0.4.5 - 2017/04/02
+
+- Bump up google-cloud-pubsub to v0.24.x
+
+### Release 0.4.4 - 2017/03/07
+
+- Bump up google-cloud-pubsub to v0.23.x
+
+### Release 0.4.3 - 2017/02/16
+
+- Input plugin
+  - Add "status" method to the http rpc api
+
+### Release 0.4.2 - 2017/02/03
+
+- Make retry to get topic/subscription when Pub/Sub API returns 50x code
+
+### Release 0.4.1 - 2017/02/02
+
+- Bump up google-cloud-pubsub to v0.22.x
+- Input plugin
+  - Rescue 50x errors on acknowledge api
+
+### Release 0.4.0 - 2017/01/21
+
+- Input plugin
+  - Add feature to use record key as tag
+
+### Release 0.3.4 - 2017/01/03
+
+- Output plugin
+  - Rescue 50x errors
+- Input plugin
+  - Guard emit to be called with multi-threading
+  - Rescue 50x errors
+  - Enabled to select whether to raise an exception if message processing failed
+
+### Release 0.3.3 - 2016/12/03
+
+- Input plugin
+  - Fix undefined variable error
+
+### Release 0.3.2 - 2016/11/13
+
+- Add plugin param desc
+- Input plugin
+  - Improve handling to acknowledge messages
+
+### Release 0.3.1 - 2016/11/03
+
+- Output plugin
+  - Improve error handling
+
+### Release 0.3.0 - 2016/10/30
+
+- Bump up google-cloud-pubsub to v0.21
+- Input plugin
+  - Add multithreaded pulling feature
+
+### Release 0.2.0 - 2016/10/15
+
+- Input plugin
+  - Add HTTP RPC feature
+
+### Release 0.1.4 - 2016/09/19
+
+- Input plugin
+  - `pull_interval` can be specified float value
+  - `topic` must be specified
+
+### Release 0.1.3 - 2016/09/17
+
+- Input plugin
+  - Fix error handling and add debug logging
+
+### Release 0.1.2 - 2016/09/11
+
+- Output plugin
+  - Change default max message size and add debug message
+
+### Release 0.1.1 - 2016/08/27
+
+- Bump up google-cloud-pubsub (gcloud-ruby) to 0.20
+
+### Release 0.1.0 - 2016/08/22
+
+- Use formatter / parser plugin and add format configuration
+- Bump up gcloud-ruby to 0.12
+- Remove dependency on lightening buffer
+- Fix error caused by Pub/Sub quotas

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 mdoi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,250 @@
+# fluent-plugin-gcloud-pubsub-custom
+
+[![Build Status](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom.svg?branch=master)](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom.svg)](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
+
+This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
+
+## Overview
+
+[Google Cloud Pub/Sub](https://cloud.google.com/pubsub/) Input/Output(BufferedOutput) plugin for [Fluentd](http://www.fluentd.org/) with [google-cloud](https://googlecloudplatform.github.io/google-cloud-ruby/) gem
+
+- Publish messages to Google Cloud Pub/Sub
+- Pull messages from Google Cloud Pub/Sub
+
+## Preparation
+
+- Create a project on Google Developer Console
+- Add a topic of Cloud Pub/Sub to the project
+- Add a pull style subscription to the topic
+- Download your credential (json) or [set scope on GCE instance](https://cloud.google.com/compute/docs/api/how-tos/authorization)
+
+When using output plugin, you need to grant Pub/Sub Publisher and Pub/Sub Viewer role to IAM.
+
+Also, when using input plugin, you need to grant Pub/Sub Subscriber and Pub/Sub Viewer role to IAM.
+
+## Requirements
+
+| fluent-plugin-gcloud-pubsub-custom | fluentd | ruby |
+|------------------------|---------|------|
+| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
+|  < 1.0.0 | >= v0.12.0 | >= 1.9 |
+
+## Installation
+
+Install by gem:
+
+```shell
+$ gem install fluent-plugin-gcloud-pubsub-custom
+```
+
+**Caution**
+
+This plugin doesn't work in [td-agent](http://docs.fluentd.org/articles/install-by-rpm).
+
+Please use in [Fluentd installed by gem](http://docs.fluentd.org/articles/install-by-gem).
+
+## Configuration
+
+### Publish messages
+
+Use `gcloud_pubsub` output plugin.
+
+```
+<match example.publish>
+  @type gcloud_pubsub
+  project <YOUR PROJECT>
+  key <YOUR KEY>
+  topic <YOUR TOPIC>
+  autocreate_topic false
+  max_messages 1000
+  max_total_size 9800000
+  max_message_size 4000000
+  compress_batches false
+  <buffer>
+    @type memory
+    flush_interval 1s
+  </buffer>
+  <format>
+    @type json
+  </format>
+</match>
+```
+
+- `project` (optional)
+  - Set your GCP project.
+  - Running fluentd on GCP, you don't have to specify.
+  - You can also use environment variable such as `GCLOUD_PROJECT`.
+- `key` (optional)
+  - Set your credential file path.
+  - Running fluentd on GCP, you can use scope instead of specifying this.
+  - You can also use environment variable such as `GCLOUD_KEYFILE`.
+- `topic` (required)
+  - Set topic name to publish.
+  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
+- `autocreate_topic` (optional, default: `false`)
+  - If set to `true`, specified topic will be created when it doesn't exist.
+- `max_messages` (optional, default: `1000`)
+  - Publishing messages count per request to Cloud Pub/Sub.
+    - See https://cloud.google.com/pubsub/quotas#other_limits
+- `max_total_size` (optional, default: `9800000` = `9.8MB`)
+  - Publishing messages bytesize per request to Cloud Pub/Sub. This parameter affects only message size. You should specify a little smaller value than quota.
+    - See https://cloud.google.com/pubsub/quotas#other_limits
+- `max_message_size` (optional, default: `4000000` = `4MB`)
+  - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
+- `attribute_keys` (optional, default: `[]`)
+  - Extract these fields from the record and send them as attributes on the Pub/Sub message. Cannot be set if compress_batches is enabled.
+- `metric_prefix` (optional, default: `fluentd_output_gcloud_pubsub`)
+  - The prefix for Prometheus metric names
+- `compress_batches` (optional, default: `false`)
+  - If set to `true`, messages will be batched and compressed before publication. See [message compression](#message-compression) for details.
+
+### Pull messages
+
+Use `gcloud_pubsub` input plugin.
+
+```
+<source>
+  @type gcloud_pubsub
+  tag example.pull
+  project <YOUR PROJECT>
+  key <YOUR KEY>
+  topic <YOUR TOPIC>
+  subscription <YOUR SUBSCRIPTION>
+  max_messages 1000
+  return_immediately true
+  pull_interval 0.5
+  pull_threads 2
+  parse_error_action exception
+  enable_rpc true
+  rpc_bind 0.0.0.0
+  rpc_port 24680
+  <parse>
+    @type json
+  </parse>
+</source>
+```
+
+- `tag` (required)
+  - Set tag of messages.
+  - If `tag_key` is specified, `tag` is used as tag when record don't have specified key.
+- `tag_key` (optional)
+  - Set key to be used as tag.
+- `project` (optional)
+  - Set your GCP project
+  - Running fluentd on GCP, you don't have to specify.
+  - You can also use environment variable such as `GCLOUD_PROJECT`.
+- `key` (optional)
+  - Set your credential file path.
+  - Running fluentd on GCP, you can use scope instead of specifying this.
+  - You can also use environment variable such as `GCLOUD_KEYFILE`.
+- `topic` (optional)
+  - Set topic name that the subscription belongs to.
+- `subscription` (required)
+  - Set subscription name to pull.
+- `max_messages` (optional, default: `100`)
+  - See maxMessages on https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull
+- `return_immediately` (optional, default: `true`)
+  - See returnImmediately on https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull
+  - If `return_immediately` is `true` or pulling message is stopped by HTTP RPC, this plugin wait `pull_interval` each pull.
+- `pull_interval` (optional, default: `5.0`)
+  - Pulling messages by intervals of specified seconds.
+- `pull_threads` (optional, default: `1`)
+  - Set number of threads to pull messages.
+- `attribute_keys` (optional, default: `[]`)
+  - Acquire these fields from attributes on the Pub/Sub message and merge them into the record.
+- `parse_error_action` (optional, default: `exception`)
+  - Set error type when parsing messages fails.
+    - `exception`: Raise exception. Messages are not acknowledged.
+    - `warning`: Only logging as warning.
+- `metric_prefix` (optional, default: `fluentd_input_gcloud_pubsub`)
+  - The prefix for Prometheus metric names
+- `enable_rpc` (optional, default: `false`)
+  - If `true` is specified, HTTP RPC to stop or start pulling message is enabled.
+- `rpc_bind` (optional, default: `0.0.0.0`)
+  - Bind IP address for HTTP RPC.
+- `rpc_port` (optional, default: `24680`)
+  - Port for HTTP RPC.
+
+## Message compression
+
+The `compress_batches` option can be used to enable the compression of messages
+_before_ publication to Pub/Sub.
+
+This works by collecting the buffered messages, taking up to `max_total_size` or
+`max_message_size` input records, then compressing them with Zlib (i.e.
+gzip/Deflate) before publishing them as a single message to the Pub/Sub topic.
+
+When transporting large volumes of records via Pub/Sub, e.g. multiple Terabytes
+per month, this can lead to significant cost savings, as typically the CPU time
+required to compress the messages will be minimal in comparison to the Pub/Sub
+costs.
+
+The compression ratio achievable will vary largely depending on the homogeneity
+of the input records, but typically will be 50% at the very minimum and often
+around 80-90%.
+
+In order to achieve good compression, consider the following:
+- Ensure that the buffer is being filled with a reasonable batch of messages: do
+  not use `flush_mode immediate`, and keep the `flush_interval` value
+  sufficiently high. Use the Prometheus metrics to determine how many records
+  are being published per message.
+- Keep the `max_messages` and `max_message_size` values high (the defaults are
+  optimal).
+- If there are many different sources of messages being mixed and routed to a
+  single `gcloud_pubsub` output, use multiple outputs (which will each have
+  their own buffer) through tagging or [labelling][fluentd-labels].
+
+[fluentd-labels]: https://docs.fluentd.org/quickstart/life-of-a-fluentd-event#labels
+
+The receiving end must be able to decode these compressed batches of messages,
+which it can determine via an attribute set on the Pub/Sub message. The
+`gcloud_pubsub` input plugin will do this transparently, decompressing any
+messages which contain a batch of records and normally processing any messages
+which represent just a single record.
+Therefore, as long as all of the receivers are updated with support for
+compressed batches first, it's then possible to gradually roll out this feature.
+
+## Prometheus metrics
+
+The input and output plugins expose several metrics in order to monitor
+performance:
+
+- `fluentd_output_gcloud_pubsub_compression_enabled`
+  - Gauge: Whether compression/batching is enabled
+- `fluentd_output_gcloud_pubsub_messages_published_per_batch`
+  - Histogram: Number of records published to Pub/Sub per buffer flush
+- `fluentd_output_gcloud_pubsub_messages_published_bytes`
+  - Histogram: Total size in bytes of the records published to Pub/Sub,
+    **before** compression.
+- `fluentd_output_gcloud_pubsub_messages_compression_duration_seconds`
+  - Histogram: Time taken to compress a batch of messages
+- `fluentd_output_gcloud_pubsub_messages_compressed_size_per_original_size_ratio`
+  - Histogram: Compression ratio achieved on a batch of messages, expressed in
+    terms of space saved.
+
+- `fluentd_input_gcloud_pubsub_pull_errors_total`
+  - Counter: Errors encountered while pulling or processing messages (split by a
+    `retryable` label)
+- `fluentd_input_gcloud_pubsub_messages_pulled`
+  - Histogram: Number of Pub/Sub messages pulled by the subscriber on each invocation
+- `fluentd_input_gcloud_pubsub_messages_pulled_bytes`
+  - Histogram: Total size in bytes of the Pub/Sub messages pulled by the subscriber on each invocation
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## TODO
+
+- Add `tag` attribute in output plugin and use `tag` attribute as tag in input plugin.
+- Send ack after other output plugin committed (if possible).
+
+## Authors
+
+- [@mdoi](https://github.com/mdoi) : First author
+- [@mia-0032](https://github.com/mia-0032) : This version author