fluent-plugin-gcloud-pubsub-custom 1.3.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 940568295dd0772bb0e7f0fb2537c6fff1b95f6fcc2e8716b578c512529e808f
4
- data.tar.gz: 9f2b817300f920541942d03a207290a833a945613611b0a25e72db52f39f9a16
3
+ metadata.gz: 3bd7e53135bef7ce1f7ce3796659b6deb1ebaf6ff303755a84fdd886c4f46bf3
4
+ data.tar.gz: a04ddb36d5c0bceb7ac1fb8947b537ee0c38b9b26f6b5d5c53bb2ed034deb4e0
5
5
  SHA512:
6
- metadata.gz: 2911ef120d13bf8258981afa08d3124a01c04e88bf6af25f873c54f2eec1220f22b6473b24cfc6bbf750321933b28488c2f80a7a1b7459f9faa846044dca5e30
7
- data.tar.gz: 8ce43148345570ef849d32097da05648daeae198feac5794c15cabb9272e0471d0add6409ae2a1889d2eb5e812786618eeab5c4d6317ae7eff6e05ad4d333ce0
6
+ metadata.gz: f0d6537d00021bb6d849ff7c9c9e8b4b233214826b098db649820a0a310d7d60170a8642db3b7b7acee2f83b70a392ad6a9ef8adbb47f22bc2d1be2022129d03
7
+ data.tar.gz: 2b742512024102940dd447f6450377684a1834c89582e38d6498957dc6219c5625afa49320f2c6c0139ff73bd7e98ad61085029b0c4794e3418c0324b0346bc0
@@ -0,0 +1,25 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+ branches: [ master ]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ ruby-version: ['2.4', '2.5', '2.6', '2.7']
16
+
17
+ steps:
18
+ - uses: actions/checkout@v2
19
+ - name: Set up Ruby
20
+ uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: ${{ matrix.ruby-version }}
23
+ bundler-cache: true
24
+ - name: Run tests
25
+ run: bundle exec rake test
data/CHANGELOG.md CHANGED
@@ -1,5 +1,34 @@
1
1
  ## ChangeLog
2
2
 
3
+ ### Release 1.6.0 - 2021/06/17
4
+
5
+ - Output plugin
6
+ - Add `timeout` feature
7
+ - Add `attribute_key_values` feature
8
+
9
+ ### Release 1.5.0 - 2021/01/26
10
+
11
+ - Bump up google-cloud-pubsub to v2.3.x
12
+ - Add `endpoint` parameter
13
+ - Add gzip compression/decompression feature
14
+
15
+ ### Release 1.4.0 - 2020/08/11
16
+
17
+ - Bump up google-cloud-pubsub to v0.39.x
18
+ - In preparation for updating to v1.x
19
+ - Output plugin
20
+ - Add `dest_project` parameter
21
+
22
+ ### Release 1.3.2 - 2019/08/16
23
+
24
+ - Input plugin
25
+ - Make `topic` parameter optional
26
+
27
+ ### Release 1.3.1 - 2018/06/08
28
+
29
+ - Input plugin
30
+ - Dumps raw string to logs for parsing errors
31
+
3
32
  ### Release 1.3.0 - 2018/05/18
4
33
 
5
34
  - Make attributes available
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # fluent-plugin-gcloud-pubsub-custom
2
2
 
3
- [![Build Status](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom.svg?branch=master)](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom)
3
+ ![Test](https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom/workflows/Test/badge.svg)
4
4
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom.svg)](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
5
5
 
6
6
  This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
@@ -60,6 +60,8 @@ Use `gcloud_pubsub` output plugin.
60
60
  max_messages 1000
61
61
  max_total_size 9800000
62
62
  max_message_size 4000000
63
+ endpoint <Endpoint URL>
64
+ timeout 60
63
65
  <buffer>
64
66
  @type memory
65
67
  flush_interval 1s
@@ -81,6 +83,8 @@ Use `gcloud_pubsub` output plugin.
81
83
  - `topic` (required)
82
84
  - Set topic name to publish.
83
85
  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
86
+ - `dest_project` (optional, default: `nil`)
87
+ - Set your destination GCP project if you publish messages cross project.
84
88
  - `autocreate_topic` (optional, default: `false`)
85
89
  - If set to `true`, specified topic will be created when it doesn't exist.
86
90
  - `max_messages` (optional, default: `1000`)
@@ -92,7 +96,15 @@ Use `gcloud_pubsub` output plugin.
92
96
  - `max_message_size` (optional, default: `4000000` = `4MB`)
93
97
  - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
94
98
  - `attribute_keys` (optional, default: `[]`)
95
- - Publishing the set fields as attributes.
99
+ - Publishing the set fields as attributes generated from input message.
100
+ - `attribute_key_values` (optional, default: `{}`)
101
+ - Publishing the set fields as attributes generated from input params
102
+ - `endpoint`(optional)
103
+ - Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
104
+ - `compression` (optional, default: `nil`)
105
+ - If set to `gzip`, messages will be compressed with gzip.
106
+ - `timeout` (optional)
107
+ - Set default timeout to use in publish requests.
96
108
 
97
109
  ### Pull messages
98
110
 
@@ -133,8 +145,8 @@ Use `gcloud_pubsub` input plugin.
133
145
  - Set your credential file path.
134
146
  - Running fluentd on GCP, you can use scope instead of specifying this.
135
147
  - You can also use environment variable such as `GCLOUD_KEYFILE`.
136
- - `topic` (required)
137
- - Set topic name to pull.
148
+ - `topic` (optional)
149
+ - Set topic name that the subscription belongs to.
138
150
  - `subscription` (required)
139
151
  - Set subscription name to pull.
140
152
  - `max_messages` (optional, default: `100`)
@@ -158,6 +170,8 @@ Use `gcloud_pubsub` input plugin.
158
170
  - Bind IP address for HTTP RPC.
159
171
  - `rpc_port` (optional, default: `24680`)
160
172
  - Port for HTTP RPC.
173
+ - `decompression` (optional, default: `nil`)
174
+ - If set to `gzip`, messages will be decompressed with gzip.
161
175
 
162
176
  ## Contributing
163
177
 
data/Rakefile CHANGED
@@ -7,6 +7,7 @@ Rake::TestTask.new(:test) do |test|
7
7
  test.libs << 'lib' << 'test'
8
8
  test.test_files = FileList['test/plugin/test_*.rb']
9
9
  test.verbose = true
10
+ test.warning = false
10
11
  end
11
12
 
12
13
  task :default => [:build]
@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
7
7
  gem.license = "MIT"
8
8
  gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
9
9
  gem.summary = gem.description
10
- gem.version = "1.3.0"
10
+ gem.version = "1.6.0"
11
11
  gem.authors = ["Yoshihiro MIYAI"]
12
12
  gem.email = "msparrow17@gmail.com"
13
- gem.has_rdoc = false
14
13
  gem.files = `git ls-files`.split("\n")
15
14
  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
15
  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
16
  gem.require_paths = ['lib']
18
17
 
19
18
  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
20
- gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.30.0"
19
+ gem.add_runtime_dependency "google-cloud-pubsub", "~> 2.3.1"
21
20
 
22
21
  gem.add_development_dependency "bundler"
23
22
  gem.add_development_dependency "rake"
@@ -24,18 +24,23 @@ module Fluent
24
24
  end
25
25
 
26
26
  class Publisher
27
- def initialize(project, key, autocreate_topic)
28
- @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
27
+ def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout)
28
+ @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout
29
29
  @autocreate_topic = autocreate_topic
30
+ @dest_project = dest_project
30
31
  @topics = {}
31
32
  end
32
33
 
33
34
  def topic(topic_name)
34
35
  return @topics[topic_name] if @topics.has_key? topic_name
35
36
 
36
- client = @pubsub.topic topic_name
37
- if client.nil? && @autocreate_topic
38
- client = @pubsub.create_topic topic_name
37
+ if @dest_project.nil?
38
+ client = @pubsub.topic topic_name
39
+ if client.nil? && @autocreate_topic
40
+ client = @pubsub.create_topic topic_name
41
+ end
42
+ else
43
+ client = @pubsub.topic topic_name, project: @dest_project
39
44
  end
40
45
  if client.nil?
41
46
  raise Error.new "topic:#{topic_name} does not exist."
@@ -59,8 +64,12 @@ module Fluent
59
64
  class Subscriber
60
65
  def initialize(project, key, topic_name, subscription_name)
61
66
  pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
62
- topic = pubsub.topic topic_name
63
- @client = topic.subscription subscription_name
67
+ if topic_name.nil?
68
+ @client = pubsub.subscription subscription_name
69
+ else
70
+ topic = pubsub.topic topic_name
71
+ @client = topic.subscription subscription_name
72
+ end
64
73
  raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
65
74
  end
66
75
 
@@ -1,6 +1,7 @@
1
1
  require 'json'
2
2
  require 'webrick'
3
3
 
4
+ require 'fluent/plugin/compressable'
4
5
  require 'fluent/plugin/input'
5
6
  require 'fluent/plugin/parser'
6
7
 
@@ -8,6 +9,8 @@ require 'fluent/plugin/gcloud_pubsub/client'
8
9
 
9
10
  module Fluent::Plugin
10
11
  class GcloudPubSubInput < Input
12
+ include Fluent::Plugin::Compressable
13
+
11
14
  Fluent::Plugin.register_input('gcloud_pubsub', self)
12
15
 
13
16
  helpers :compat_parameters, :parser, :thread
@@ -26,7 +29,7 @@ module Fluent::Plugin
26
29
  desc 'Set your credential file path.'
27
30
  config_param :key, :string, default: nil
28
31
  desc 'Set topic name to pull.'
29
- config_param :topic, :string
32
+ config_param :topic, :string, default: nil
30
33
  desc 'Set subscription name to pull.'
31
34
  config_param :subscription, :string
32
35
  desc 'Pulling messages by intervals of specified seconds.'
@@ -48,6 +51,8 @@ module Fluent::Plugin
48
51
  config_param :rpc_bind, :string, default: '0.0.0.0'
49
52
  desc 'Port for HTTP RPC.'
50
53
  config_param :rpc_port, :integer, default: 24680
54
+ desc 'Decompress messages'
55
+ config_param :decompression, :string, default: nil
51
56
 
52
57
  config_section :parse do
53
58
  config_set_default :@type, DEFAULT_PARSER_TYPE
@@ -114,6 +119,11 @@ module Fluent::Plugin
114
119
  end
115
120
 
116
121
  @parser = parser_create
122
+ @decompress = if @decompression == 'gzip'
123
+ method(:gzip_decompress)
124
+ else
125
+ method(:no_decompress)
126
+ end
117
127
  end
118
128
 
119
129
  def start
@@ -168,6 +178,14 @@ module Fluent::Plugin
168
178
  record.delete(@tag_key) || @tag
169
179
  end
170
180
 
181
+ def gzip_decompress(message)
182
+ decompress message
183
+ end
184
+
185
+ def no_decompress(message)
186
+ message
187
+ end
188
+
171
189
  def start_rpc
172
190
  log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
173
191
  @rpc_srv = WEBrick::HTTPServer.new(
@@ -221,7 +239,7 @@ module Fluent::Plugin
221
239
  end
222
240
 
223
241
  messages.each do |m|
224
- line = m.message.data.chomp
242
+ line = @decompress.call(m.message.data).chomp
225
243
  attributes = m.attributes
226
244
  @parser.parse(line) do |time, record|
227
245
  if time && record
@@ -233,9 +251,9 @@ module Fluent::Plugin
233
251
  else
234
252
  case @parse_error_action
235
253
  when :exception
236
- raise FailedParseError.new "pattern not match: #{line.inspect}"
254
+ raise FailedParseError.new "pattern not match: #{line}"
237
255
  else
238
- log.warn 'pattern not match', record: line.inspect
256
+ log.warn 'pattern not match', record: line
239
257
  end
240
258
  end
241
259
  end
@@ -1,9 +1,11 @@
1
+ require 'fluent/plugin/compressable'
1
2
  require 'fluent/plugin/output'
2
3
  require 'fluent/plugin/gcloud_pubsub/client'
3
4
  require 'fluent/plugin_helper/inject'
4
5
 
5
6
  module Fluent::Plugin
6
7
  class GcloudPubSubOutput < Output
8
+ include Fluent::Plugin::Compressable
7
9
  include Fluent::PluginHelper::Inject
8
10
 
9
11
  Fluent::Plugin.register_output('gcloud_pubsub', self)
@@ -19,6 +21,8 @@ module Fluent::Plugin
19
21
  config_param :key, :string, :default => nil
20
22
  desc 'Set topic name to publish.'
21
23
  config_param :topic, :string
24
+ desc "Set your dest GCP project if publishing cross project"
25
+ config_param :dest_project, :string, :default => nil
22
26
  desc "If set to `true`, specified topic will be created when it doesn't exist."
23
27
  config_param :autocreate_topic, :bool, :default => false
24
28
  desc 'Publishing messages count per request to Cloud Pub/Sub.'
@@ -27,8 +31,16 @@ module Fluent::Plugin
27
31
  config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
28
32
  desc 'Limit bytesize per message.'
29
33
  config_param :max_message_size, :integer, :default => 4000000 # 4MB
30
- desc 'Publishing the set field as an attribute'
34
+ desc 'Publishing the set field as an attribute created from input message'
31
35
  config_param :attribute_keys, :array, :default => []
36
+ desc 'Publishing the set field as an attribute created from input config params'
37
+ config_param :attribute_key_values, :hash, :default => {}
38
+ desc 'Set service endpoint'
39
+ config_param :endpoint, :string, :default => nil
40
+ desc 'Compress messages'
41
+ config_param :compression, :string, :default => nil
42
+ desc 'Set default timeout to use in publish requests'
43
+ config_param :timeout, :integer, :default => nil
32
44
 
33
45
  config_section :buffer do
34
46
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -43,11 +55,16 @@ module Fluent::Plugin
43
55
  super
44
56
  placeholder_validate!(:topic, @topic)
45
57
  @formatter = formatter_create
58
+ @compress = if @compression == 'gzip'
59
+ method(:gzip_compress)
60
+ else
61
+ method(:no_compress)
62
+ end
46
63
  end
47
64
 
48
65
  def start
49
66
  super
50
- @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
67
+ @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint, @timeout
51
68
  end
52
69
 
53
70
  def format(tag, time, record)
@@ -56,7 +73,8 @@ module Fluent::Plugin
56
73
  @attribute_keys.each do |key|
57
74
  attributes[key] = record.delete(key)
58
75
  end
59
- [@formatter.format(tag, time, record), attributes].to_msgpack
76
+ attributes.merge! @attribute_key_values
77
+ [@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
60
78
  end
61
79
 
62
80
  def formatted_to_msgpack_binary?
@@ -106,5 +124,13 @@ module Fluent::Plugin
106
124
  log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
107
125
  @publisher.publish(topic, messages)
108
126
  end
127
+
128
+ def gzip_compress(message)
129
+ compress message
130
+ end
131
+
132
+ def no_compress(message)
133
+ message
134
+ end
109
135
  end
110
136
  end
@@ -10,22 +10,14 @@ end
10
10
  require 'test/unit'
11
11
  require "test/unit/rr"
12
12
 
13
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
14
- $LOAD_PATH.unshift(File.dirname(__FILE__))
13
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
15
14
 
16
- require 'fluent/test'
17
- unless ENV.has_key?('VERBOSE')
18
- nulllogger = Object.new
19
- nulllogger.instance_eval {|obj|
20
- def method_missing(method, *args)
21
- # pass
22
- end
23
- }
24
- $log = nulllogger
25
- end
15
+ require "fluent/test"
16
+ require "fluent/test/driver/output"
17
+ require "fluent/test/helpers"
26
18
 
27
19
  require 'fluent/plugin/in_gcloud_pubsub'
28
20
  require 'fluent/plugin/out_gcloud_pubsub'
29
21
 
30
- class Test::Unit::TestCase
31
- end
22
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
23
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -1,7 +1,8 @@
1
1
  require 'net/http'
2
2
  require 'json'
3
3
 
4
- require_relative "../test_helper"
4
+ require_relative "../helper"
5
+ require 'fluent/plugin/compressable'
5
6
  require "fluent/test/driver/input"
6
7
 
7
8
  class GcloudPubSubInputTest < Test::Unit::TestCase
@@ -164,6 +165,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
164
165
  end
165
166
  end
166
167
 
168
+ class DummyMsgDataGzipped
169
+ include Fluent::Plugin::Compressable
170
+
171
+ def data
172
+ compress '{"foo": "bar"}'
173
+ end
174
+ end
175
+ class DummyMessageGzipped
176
+ def message
177
+ DummyMsgDataGzipped.new
178
+ end
179
+ def attributes
180
+ {"attr_1" => "a", "attr_2" => "b"}
181
+ end
182
+ end
183
+
167
184
  setup do
168
185
  @subscriber = mock!
169
186
  @topic_mock = mock!.subscription('subscription-test') { @subscriber }
@@ -263,6 +280,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
263
280
  end
264
281
  end
265
282
 
283
+ test 'with gzip decompression' do
284
+ messages = Array.new(1, DummyMessageGzipped.new)
285
+ @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
286
+ @subscriber.acknowledge(messages).at_least(1)
287
+
288
+ d = create_driver("#{CONFIG}\ndecompression gzip")
289
+ d.run(expect_emits: 1, timeout: 3)
290
+ emits = d.events
291
+
292
+ assert_equal(1, emits.length)
293
+ emits.each do |tag, time, record|
294
+ assert_equal("test", tag)
295
+ assert_equal({"foo" => "bar"}, record)
296
+ end
297
+ end
298
+
266
299
  test 'invalid messages with parse_error_action warning' do
267
300
  messages = Array.new(1, DummyInvalidMessage.new)
268
301
  @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
@@ -1,10 +1,14 @@
1
1
  # coding: utf-8
2
- require_relative "../test_helper"
2
+ require_relative "../helper"
3
+ require 'fluent/plugin/compressable'
3
4
  require "fluent/test/driver/output"
4
5
  require "fluent/test/helpers"
6
+ require 'json'
7
+ require 'msgpack'
5
8
 
6
9
  class GcloudPubSubOutputTest < Test::Unit::TestCase
7
10
  include Fluent::Test::Helpers
11
+ include Fluent::Plugin::Compressable
8
12
 
9
13
  CONFIG = %[
10
14
  project project-test
@@ -62,6 +66,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
62
66
 
63
67
  assert_equal(true, d.instance.autocreate_topic)
64
68
  end
69
+
70
+ test '"dest_project" can be specified' do
71
+ d = create_driver(%[
72
+ project project-test
73
+ topic topic-test
74
+ key key-test
75
+ dest_project dest-project-test
76
+ ])
77
+
78
+ assert_equal("dest-project-test", d.instance.dest_project)
79
+ end
65
80
  end
66
81
 
67
82
  sub_test_case 'topic' do
@@ -69,6 +84,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
69
84
  @publisher = mock!
70
85
  @pubsub_mock = mock!
71
86
  stub(Google::Cloud::Pubsub).new { @pubsub_mock }
87
+
88
+ @old_report_on_exception = Thread.report_on_exception
89
+ Thread.report_on_exception = false
90
+ end
91
+
92
+ teardown do
93
+ Thread.report_on_exception = @old_report_on_exception
72
94
  end
73
95
 
74
96
  test '"autocreate_topic" is enabled' do
@@ -87,6 +109,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
87
109
  end
88
110
  end
89
111
 
112
+ test '"dest_project" is set' do
113
+ d = create_driver(%[
114
+ project project-test
115
+ topic topic-test
116
+ key key-test
117
+ dest_project dest-project-test
118
+ ])
119
+
120
+ @publisher.publish.once
121
+ @pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
122
+ d.run(default_tag: "test") do
123
+ d.feed(@time, {"a" => "b"})
124
+ end
125
+ end
126
+
90
127
  test '40x error occurred on connecting to Pub/Sub' do
91
128
  d = create_driver
92
129
 
@@ -147,6 +184,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
147
184
  @publisher = mock!
148
185
  @pubsub_mock = mock!.topic(anything) { @publisher }
149
186
  stub(Google::Cloud::Pubsub).new { @pubsub_mock }
187
+
188
+ @old_report_on_exception = Thread.report_on_exception
189
+ Thread.report_on_exception = false
190
+ end
191
+
192
+ teardown do
193
+ Thread.report_on_exception = @old_report_on_exception
150
194
  end
151
195
 
152
196
  test 'messages are divided into "max_messages"' do
@@ -222,5 +266,83 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
222
266
  end
223
267
  assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
224
268
  end
269
+
270
+ test 'with gzip compression' do
271
+ d = create_driver(%[
272
+ project project-test
273
+ topic topic-test
274
+ key key-test
275
+ compression gzip
276
+ ])
277
+ test_msg = {"a" => "abc"}
278
+
279
+ @publisher.publish.once
280
+ d.run(default_tag: "test") do
281
+ d.feed(test_msg)
282
+ end
283
+
284
+ assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
285
+ end
286
+
287
+ test 'with attribute_keys' do
288
+ d = create_driver(%[
289
+ project project-test
290
+ topic topic-test
291
+ key key-test
292
+ attribute_keys a,b
293
+ ])
294
+
295
+ @publisher.publish.once
296
+ d.run(default_tag: "test") do
297
+ d.feed(@time, {"a" => "foo", "b" => "bar", "c" => "baz"})
298
+ end
299
+
300
+ formatted = MessagePack.unpack(d.formatted[0])
301
+ message = JSON.parse(formatted[0])
302
+ assert_equal(message, {"c" => "baz"})
303
+ attributes = formatted[1]
304
+ assert_equal(attributes, {"a" => "foo", "b" => "bar"})
305
+ end
306
+
307
+ test 'with attribute_key_values' do
308
+ d = create_driver(%[
309
+ project project-test
310
+ topic topic-test
311
+ key key-test
312
+ attribute_key_values {"key1": "value1", "key2": "value2"}
313
+ ])
314
+
315
+ @publisher.publish.once
316
+ d.run(default_tag: "test") do
317
+ d.feed(@time, {"a" => "foo", "b" => "bar"})
318
+ end
319
+
320
+ formatted = MessagePack.unpack(d.formatted[0])
321
+ message = JSON.parse(formatted[0])
322
+ assert_equal(message, {"a" => "foo", "b" => "bar"})
323
+ attributes = formatted[1]
324
+ assert_equal(attributes, {"key1" => "value1", "key2" => "value2"})
325
+ end
326
+
327
+ test 'with attribute_keys and attribute_key_values' do
328
+ d = create_driver(%[
329
+ project project-test
330
+ topic topic-test
331
+ key key-test
332
+ attribute_keys a
333
+ attribute_key_values {"key": "value"}
334
+ ])
335
+
336
+ @publisher.publish.once
337
+ d.run(default_tag: "test") do
338
+ d.feed(@time, {"a" => "foo", "b" => "bar"})
339
+ end
340
+
341
+ formatted = MessagePack.unpack(d.formatted[0])
342
+ message = JSON.parse(formatted[0])
343
+ assert_equal(message, {"b" => "bar"})
344
+ attributes = formatted[1]
345
+ assert_equal(attributes, {"a" => "foo", "key" => "value"})
346
+ end
225
347
  end
226
348
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-gcloud-pubsub-custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yoshihiro MIYAI
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-18 00:00:00.000000000 Z
11
+ date: 2021-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -36,14 +36,14 @@ dependencies:
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 0.30.0
39
+ version: 2.3.1
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: 0.30.0
46
+ version: 2.3.1
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: bundler
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -106,8 +106,8 @@ executables: []
106
106
  extensions: []
107
107
  extra_rdoc_files: []
108
108
  files:
109
+ - ".github/workflows/test.yml"
109
110
  - ".gitignore"
110
- - ".travis.yml"
111
111
  - CHANGELOG.md
112
112
  - Gemfile
113
113
  - LICENSE
@@ -117,14 +117,14 @@ files:
117
117
  - lib/fluent/plugin/gcloud_pubsub/client.rb
118
118
  - lib/fluent/plugin/in_gcloud_pubsub.rb
119
119
  - lib/fluent/plugin/out_gcloud_pubsub.rb
120
+ - test/helper.rb
120
121
  - test/plugin/test_in_gcloud_pubsub.rb
121
122
  - test/plugin/test_out_gcloud_pubsub.rb
122
- - test/test_helper.rb
123
123
  homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
124
124
  licenses:
125
125
  - MIT
126
126
  metadata: {}
127
- post_install_message:
127
+ post_install_message:
128
128
  rdoc_options: []
129
129
  require_paths:
130
130
  - lib
@@ -139,12 +139,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
139
139
  - !ruby/object:Gem::Version
140
140
  version: '0'
141
141
  requirements: []
142
- rubyforge_project:
143
- rubygems_version: 2.7.3
144
- signing_key:
142
+ rubygems_version: 3.1.2
143
+ signing_key:
145
144
  specification_version: 4
146
145
  summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
147
146
  test_files:
147
+ - test/helper.rb
148
148
  - test/plugin/test_in_gcloud_pubsub.rb
149
149
  - test/plugin/test_out_gcloud_pubsub.rb
150
- - test/test_helper.rb
data/.travis.yml DELETED
@@ -1,25 +0,0 @@
1
- language: ruby
2
-
3
- rvm:
4
- - 2.1
5
- - 2.2
6
- - 2.3.6
7
- - 2.4.3
8
- - 2.5.0
9
- - ruby-head
10
-
11
- gemfile:
12
- - Gemfile
13
-
14
- branches:
15
- only:
16
- - master
17
-
18
- before_install: gem update bundler
19
- script: bundle exec rake test
20
-
21
- sudo: false
22
-
23
- matrix:
24
- allow_failures:
25
- - rvm: ruby-head