fluent-plugin-gcloud-pubsub-custom 1.3.0 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 940568295dd0772bb0e7f0fb2537c6fff1b95f6fcc2e8716b578c512529e808f
4
- data.tar.gz: 9f2b817300f920541942d03a207290a833a945613611b0a25e72db52f39f9a16
3
+ metadata.gz: 3bd7e53135bef7ce1f7ce3796659b6deb1ebaf6ff303755a84fdd886c4f46bf3
4
+ data.tar.gz: a04ddb36d5c0bceb7ac1fb8947b537ee0c38b9b26f6b5d5c53bb2ed034deb4e0
5
5
  SHA512:
6
- metadata.gz: 2911ef120d13bf8258981afa08d3124a01c04e88bf6af25f873c54f2eec1220f22b6473b24cfc6bbf750321933b28488c2f80a7a1b7459f9faa846044dca5e30
7
- data.tar.gz: 8ce43148345570ef849d32097da05648daeae198feac5794c15cabb9272e0471d0add6409ae2a1889d2eb5e812786618eeab5c4d6317ae7eff6e05ad4d333ce0
6
+ metadata.gz: f0d6537d00021bb6d849ff7c9c9e8b4b233214826b098db649820a0a310d7d60170a8642db3b7b7acee2f83b70a392ad6a9ef8adbb47f22bc2d1be2022129d03
7
+ data.tar.gz: 2b742512024102940dd447f6450377684a1834c89582e38d6498957dc6219c5625afa49320f2c6c0139ff73bd7e98ad61085029b0c4794e3418c0324b0346bc0
@@ -0,0 +1,25 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+ branches: [ master ]
8
+
9
+ jobs:
10
+ test:
11
+ runs-on: ubuntu-latest
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ ruby-version: ['2.4', '2.5', '2.6', '2.7']
16
+
17
+ steps:
18
+ - uses: actions/checkout@v2
19
+ - name: Set up Ruby
20
+ uses: ruby/setup-ruby@v1
21
+ with:
22
+ ruby-version: ${{ matrix.ruby-version }}
23
+ bundler-cache: true
24
+ - name: Run tests
25
+ run: bundle exec rake test
data/CHANGELOG.md CHANGED
@@ -1,5 +1,34 @@
1
1
  ## ChangeLog
2
2
 
3
+ ### Release 1.6.0 - 2021/06/17
4
+
5
+ - Output plugin
6
+ - Add `timeout` feature
7
+ - Add `attribute_key_values` feature
8
+
9
+ ### Release 1.5.0 - 2021/01/26
10
+
11
+ - Bump up google-cloud-pubsub to v2.3.x
12
+ - Add `endpoint` parameter
13
+ - Add gzip compression/decompression feature
14
+
15
+ ### Release 1.4.0 - 2020/08/11
16
+
17
+ - Bump up google-cloud-pubsub to v0.39.x
18
+ - In preparation for updating to v1.x
19
+ - Output plugin
20
+ - Add `dest_project` parameter
21
+
22
+ ### Release 1.3.2 - 2019/08/16
23
+
24
+ - Input plugin
25
+ - Make `topic` parameter optional
26
+
27
+ ### Release 1.3.1 - 2018/06/08
28
+
29
+ - Input plugin
30
+ - Dumps raw string to logs for parsing errors
31
+
3
32
  ### Release 1.3.0 - 2018/05/18
4
33
 
5
34
  - Make attributes available
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # fluent-plugin-gcloud-pubsub-custom
2
2
 
3
- [![Build Status](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom.svg?branch=master)](https://travis-ci.org/mia-0032/fluent-plugin-gcloud-pubsub-custom)
3
+ ![Test](https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom/workflows/Test/badge.svg)
4
4
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom.svg)](http://badge.fury.io/rb/fluent-plugin-gcloud-pubsub-custom)
5
5
 
6
6
  This plugin is forked from https://github.com/mdoi/fluent-plugin-gcloud-pubsub
@@ -60,6 +60,8 @@ Use `gcloud_pubsub` output plugin.
60
60
  max_messages 1000
61
61
  max_total_size 9800000
62
62
  max_message_size 4000000
63
+ endpoint <Endpoint URL>
64
+ timeout 60
63
65
  <buffer>
64
66
  @type memory
65
67
  flush_interval 1s
@@ -81,6 +83,8 @@ Use `gcloud_pubsub` output plugin.
81
83
  - `topic` (required)
82
84
  - Set topic name to publish.
83
85
  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
86
+ - `dest_project` (optional, default: `nil`)
87
+ - Set your destination GCP project if you publish messages cross project.
84
88
  - `autocreate_topic` (optional, default: `false`)
85
89
  - If set to `true`, specified topic will be created when it doesn't exist.
86
90
  - `max_messages` (optional, default: `1000`)
@@ -92,7 +96,15 @@ Use `gcloud_pubsub` output plugin.
92
96
  - `max_message_size` (optional, default: `4000000` = `4MB`)
93
97
  - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
94
98
  - `attribute_keys` (optional, default: `[]`)
95
- - Publishing the set fields as attributes.
99
+ - Publishing the set fields as attributes generated from input message.
100
+ - `attribute_key_values` (optional, default: `{}`)
101
+ - Publishing the set fields as attributes generated from input params
102
+ - `endpoint`(optional)
103
+ - Set Pub/Sub service endpoint. For more information, see [Service Endpoints](https://cloud.google.com/pubsub/docs/reference/service_apis_overview#service_endpoints)
104
+ - `compression` (optional, default: `nil`)
105
+ - If set to `gzip`, messages will be compressed with gzip.
106
+ - `timeout` (optional)
107
+ - Set default timeout to use in publish requests.
96
108
 
97
109
  ### Pull messages
98
110
 
@@ -133,8 +145,8 @@ Use `gcloud_pubsub` input plugin.
133
145
  - Set your credential file path.
134
146
  - Running fluentd on GCP, you can use scope instead of specifying this.
135
147
  - You can also use environment variable such as `GCLOUD_KEYFILE`.
136
- - `topic` (required)
137
- - Set topic name to pull.
148
+ - `topic` (optional)
149
+ - Set topic name that the subscription belongs to.
138
150
  - `subscription` (required)
139
151
  - Set subscription name to pull.
140
152
  - `max_messages` (optional, default: `100`)
@@ -158,6 +170,8 @@ Use `gcloud_pubsub` input plugin.
158
170
  - Bind IP address for HTTP RPC.
159
171
  - `rpc_port` (optional, default: `24680`)
160
172
  - Port for HTTP RPC.
173
+ - `decompression` (optional, default: `nil`)
174
+ - If set to `gzip`, messages will be decompressed with gzip.
161
175
 
162
176
  ## Contributing
163
177
 
data/Rakefile CHANGED
@@ -7,6 +7,7 @@ Rake::TestTask.new(:test) do |test|
7
7
  test.libs << 'lib' << 'test'
8
8
  test.test_files = FileList['test/plugin/test_*.rb']
9
9
  test.verbose = true
10
+ test.warning = false
10
11
  end
11
12
 
12
13
  task :default => [:build]
@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
7
7
  gem.license = "MIT"
8
8
  gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
9
9
  gem.summary = gem.description
10
- gem.version = "1.3.0"
10
+ gem.version = "1.6.0"
11
11
  gem.authors = ["Yoshihiro MIYAI"]
12
12
  gem.email = "msparrow17@gmail.com"
13
- gem.has_rdoc = false
14
13
  gem.files = `git ls-files`.split("\n")
15
14
  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
15
  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
16
  gem.require_paths = ['lib']
18
17
 
19
18
  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
20
- gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.30.0"
19
+ gem.add_runtime_dependency "google-cloud-pubsub", "~> 2.3.1"
21
20
 
22
21
  gem.add_development_dependency "bundler"
23
22
  gem.add_development_dependency "rake"
@@ -24,18 +24,23 @@ module Fluent
24
24
  end
25
25
 
26
26
  class Publisher
27
- def initialize(project, key, autocreate_topic)
28
- @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
27
+ def initialize(project, key, autocreate_topic, dest_project, endpoint, timeout)
28
+ @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key, endpoint: endpoint, timeout: timeout
29
29
  @autocreate_topic = autocreate_topic
30
+ @dest_project = dest_project
30
31
  @topics = {}
31
32
  end
32
33
 
33
34
  def topic(topic_name)
34
35
  return @topics[topic_name] if @topics.has_key? topic_name
35
36
 
36
- client = @pubsub.topic topic_name
37
- if client.nil? && @autocreate_topic
38
- client = @pubsub.create_topic topic_name
37
+ if @dest_project.nil?
38
+ client = @pubsub.topic topic_name
39
+ if client.nil? && @autocreate_topic
40
+ client = @pubsub.create_topic topic_name
41
+ end
42
+ else
43
+ client = @pubsub.topic topic_name, project: @dest_project
39
44
  end
40
45
  if client.nil?
41
46
  raise Error.new "topic:#{topic_name} does not exist."
@@ -59,8 +64,12 @@ module Fluent
59
64
  class Subscriber
60
65
  def initialize(project, key, topic_name, subscription_name)
61
66
  pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
62
- topic = pubsub.topic topic_name
63
- @client = topic.subscription subscription_name
67
+ if topic_name.nil?
68
+ @client = pubsub.subscription subscription_name
69
+ else
70
+ topic = pubsub.topic topic_name
71
+ @client = topic.subscription subscription_name
72
+ end
64
73
  raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
65
74
  end
66
75
 
@@ -1,6 +1,7 @@
1
1
  require 'json'
2
2
  require 'webrick'
3
3
 
4
+ require 'fluent/plugin/compressable'
4
5
  require 'fluent/plugin/input'
5
6
  require 'fluent/plugin/parser'
6
7
 
@@ -8,6 +9,8 @@ require 'fluent/plugin/gcloud_pubsub/client'
8
9
 
9
10
  module Fluent::Plugin
10
11
  class GcloudPubSubInput < Input
12
+ include Fluent::Plugin::Compressable
13
+
11
14
  Fluent::Plugin.register_input('gcloud_pubsub', self)
12
15
 
13
16
  helpers :compat_parameters, :parser, :thread
@@ -26,7 +29,7 @@ module Fluent::Plugin
26
29
  desc 'Set your credential file path.'
27
30
  config_param :key, :string, default: nil
28
31
  desc 'Set topic name to pull.'
29
- config_param :topic, :string
32
+ config_param :topic, :string, default: nil
30
33
  desc 'Set subscription name to pull.'
31
34
  config_param :subscription, :string
32
35
  desc 'Pulling messages by intervals of specified seconds.'
@@ -48,6 +51,8 @@ module Fluent::Plugin
48
51
  config_param :rpc_bind, :string, default: '0.0.0.0'
49
52
  desc 'Port for HTTP RPC.'
50
53
  config_param :rpc_port, :integer, default: 24680
54
+ desc 'Decompress messages'
55
+ config_param :decompression, :string, default: nil
51
56
 
52
57
  config_section :parse do
53
58
  config_set_default :@type, DEFAULT_PARSER_TYPE
@@ -114,6 +119,11 @@ module Fluent::Plugin
114
119
  end
115
120
 
116
121
  @parser = parser_create
122
+ @decompress = if @decompression == 'gzip'
123
+ method(:gzip_decompress)
124
+ else
125
+ method(:no_decompress)
126
+ end
117
127
  end
118
128
 
119
129
  def start
@@ -168,6 +178,14 @@ module Fluent::Plugin
168
178
  record.delete(@tag_key) || @tag
169
179
  end
170
180
 
181
+ def gzip_decompress(message)
182
+ decompress message
183
+ end
184
+
185
+ def no_decompress(message)
186
+ message
187
+ end
188
+
171
189
  def start_rpc
172
190
  log.info "listening http rpc server on http://#{@rpc_bind}:#{@rpc_port}/"
173
191
  @rpc_srv = WEBrick::HTTPServer.new(
@@ -221,7 +239,7 @@ module Fluent::Plugin
221
239
  end
222
240
 
223
241
  messages.each do |m|
224
- line = m.message.data.chomp
242
+ line = @decompress.call(m.message.data).chomp
225
243
  attributes = m.attributes
226
244
  @parser.parse(line) do |time, record|
227
245
  if time && record
@@ -233,9 +251,9 @@ module Fluent::Plugin
233
251
  else
234
252
  case @parse_error_action
235
253
  when :exception
236
- raise FailedParseError.new "pattern not match: #{line.inspect}"
254
+ raise FailedParseError.new "pattern not match: #{line}"
237
255
  else
238
- log.warn 'pattern not match', record: line.inspect
256
+ log.warn 'pattern not match', record: line
239
257
  end
240
258
  end
241
259
  end
@@ -1,9 +1,11 @@
1
+ require 'fluent/plugin/compressable'
1
2
  require 'fluent/plugin/output'
2
3
  require 'fluent/plugin/gcloud_pubsub/client'
3
4
  require 'fluent/plugin_helper/inject'
4
5
 
5
6
  module Fluent::Plugin
6
7
  class GcloudPubSubOutput < Output
8
+ include Fluent::Plugin::Compressable
7
9
  include Fluent::PluginHelper::Inject
8
10
 
9
11
  Fluent::Plugin.register_output('gcloud_pubsub', self)
@@ -19,6 +21,8 @@ module Fluent::Plugin
19
21
  config_param :key, :string, :default => nil
20
22
  desc 'Set topic name to publish.'
21
23
  config_param :topic, :string
24
+ desc "Set your dest GCP project if publishing cross project"
25
+ config_param :dest_project, :string, :default => nil
22
26
  desc "If set to `true`, specified topic will be created when it doesn't exist."
23
27
  config_param :autocreate_topic, :bool, :default => false
24
28
  desc 'Publishing messages count per request to Cloud Pub/Sub.'
@@ -27,8 +31,16 @@ module Fluent::Plugin
27
31
  config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
28
32
  desc 'Limit bytesize per message.'
29
33
  config_param :max_message_size, :integer, :default => 4000000 # 4MB
30
- desc 'Publishing the set field as an attribute'
34
+ desc 'Publishing the set field as an attribute created from input message'
31
35
  config_param :attribute_keys, :array, :default => []
36
+ desc 'Publishing the set field as an attribute created from input config params'
37
+ config_param :attribute_key_values, :hash, :default => {}
38
+ desc 'Set service endpoint'
39
+ config_param :endpoint, :string, :default => nil
40
+ desc 'Compress messages'
41
+ config_param :compression, :string, :default => nil
42
+ desc 'Set default timeout to use in publish requests'
43
+ config_param :timeout, :integer, :default => nil
32
44
 
33
45
  config_section :buffer do
34
46
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -43,11 +55,16 @@ module Fluent::Plugin
43
55
  super
44
56
  placeholder_validate!(:topic, @topic)
45
57
  @formatter = formatter_create
58
+ @compress = if @compression == 'gzip'
59
+ method(:gzip_compress)
60
+ else
61
+ method(:no_compress)
62
+ end
46
63
  end
47
64
 
48
65
  def start
49
66
  super
50
- @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
67
+ @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project, @endpoint, @timeout
51
68
  end
52
69
 
53
70
  def format(tag, time, record)
@@ -56,7 +73,8 @@ module Fluent::Plugin
56
73
  @attribute_keys.each do |key|
57
74
  attributes[key] = record.delete(key)
58
75
  end
59
- [@formatter.format(tag, time, record), attributes].to_msgpack
76
+ attributes.merge! @attribute_key_values
77
+ [@compress.call(@formatter.format(tag, time, record)), attributes].to_msgpack
60
78
  end
61
79
 
62
80
  def formatted_to_msgpack_binary?
@@ -106,5 +124,13 @@ module Fluent::Plugin
106
124
  log.debug "send message topic:#{topic} length:#{messages.length} size:#{messages.map(&:bytesize).inject(:+)}"
107
125
  @publisher.publish(topic, messages)
108
126
  end
127
+
128
+ def gzip_compress(message)
129
+ compress message
130
+ end
131
+
132
+ def no_compress(message)
133
+ message
134
+ end
109
135
  end
110
136
  end
@@ -10,22 +10,14 @@ end
10
10
  require 'test/unit'
11
11
  require "test/unit/rr"
12
12
 
13
- $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
14
- $LOAD_PATH.unshift(File.dirname(__FILE__))
13
+ $LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
15
14
 
16
- require 'fluent/test'
17
- unless ENV.has_key?('VERBOSE')
18
- nulllogger = Object.new
19
- nulllogger.instance_eval {|obj|
20
- def method_missing(method, *args)
21
- # pass
22
- end
23
- }
24
- $log = nulllogger
25
- end
15
+ require "fluent/test"
16
+ require "fluent/test/driver/output"
17
+ require "fluent/test/helpers"
26
18
 
27
19
  require 'fluent/plugin/in_gcloud_pubsub'
28
20
  require 'fluent/plugin/out_gcloud_pubsub'
29
21
 
30
- class Test::Unit::TestCase
31
- end
22
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
23
+ Test::Unit::TestCase.extend(Fluent::Test::Helpers)
@@ -1,7 +1,8 @@
1
1
  require 'net/http'
2
2
  require 'json'
3
3
 
4
- require_relative "../test_helper"
4
+ require_relative "../helper"
5
+ require 'fluent/plugin/compressable'
5
6
  require "fluent/test/driver/input"
6
7
 
7
8
  class GcloudPubSubInputTest < Test::Unit::TestCase
@@ -164,6 +165,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
164
165
  end
165
166
  end
166
167
 
168
+ class DummyMsgDataGzipped
169
+ include Fluent::Plugin::Compressable
170
+
171
+ def data
172
+ compress '{"foo": "bar"}'
173
+ end
174
+ end
175
+ class DummyMessageGzipped
176
+ def message
177
+ DummyMsgDataGzipped.new
178
+ end
179
+ def attributes
180
+ {"attr_1" => "a", "attr_2" => "b"}
181
+ end
182
+ end
183
+
167
184
  setup do
168
185
  @subscriber = mock!
169
186
  @topic_mock = mock!.subscription('subscription-test') { @subscriber }
@@ -263,6 +280,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
263
280
  end
264
281
  end
265
282
 
283
+ test 'with gzip decompression' do
284
+ messages = Array.new(1, DummyMessageGzipped.new)
285
+ @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
286
+ @subscriber.acknowledge(messages).at_least(1)
287
+
288
+ d = create_driver("#{CONFIG}\ndecompression gzip")
289
+ d.run(expect_emits: 1, timeout: 3)
290
+ emits = d.events
291
+
292
+ assert_equal(1, emits.length)
293
+ emits.each do |tag, time, record|
294
+ assert_equal("test", tag)
295
+ assert_equal({"foo" => "bar"}, record)
296
+ end
297
+ end
298
+
266
299
  test 'invalid messages with parse_error_action warning' do
267
300
  messages = Array.new(1, DummyInvalidMessage.new)
268
301
  @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
@@ -1,10 +1,14 @@
1
1
  # coding: utf-8
2
- require_relative "../test_helper"
2
+ require_relative "../helper"
3
+ require 'fluent/plugin/compressable'
3
4
  require "fluent/test/driver/output"
4
5
  require "fluent/test/helpers"
6
+ require 'json'
7
+ require 'msgpack'
5
8
 
6
9
  class GcloudPubSubOutputTest < Test::Unit::TestCase
7
10
  include Fluent::Test::Helpers
11
+ include Fluent::Plugin::Compressable
8
12
 
9
13
  CONFIG = %[
10
14
  project project-test
@@ -62,6 +66,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
62
66
 
63
67
  assert_equal(true, d.instance.autocreate_topic)
64
68
  end
69
+
70
+ test '"dest_project" can be specified' do
71
+ d = create_driver(%[
72
+ project project-test
73
+ topic topic-test
74
+ key key-test
75
+ dest_project dest-project-test
76
+ ])
77
+
78
+ assert_equal("dest-project-test", d.instance.dest_project)
79
+ end
65
80
  end
66
81
 
67
82
  sub_test_case 'topic' do
@@ -69,6 +84,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
69
84
  @publisher = mock!
70
85
  @pubsub_mock = mock!
71
86
  stub(Google::Cloud::Pubsub).new { @pubsub_mock }
87
+
88
+ @old_report_on_exception = Thread.report_on_exception
89
+ Thread.report_on_exception = false
90
+ end
91
+
92
+ teardown do
93
+ Thread.report_on_exception = @old_report_on_exception
72
94
  end
73
95
 
74
96
  test '"autocreate_topic" is enabled' do
@@ -87,6 +109,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
87
109
  end
88
110
  end
89
111
 
112
+ test '"dest_project" is set' do
113
+ d = create_driver(%[
114
+ project project-test
115
+ topic topic-test
116
+ key key-test
117
+ dest_project dest-project-test
118
+ ])
119
+
120
+ @publisher.publish.once
121
+ @pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
122
+ d.run(default_tag: "test") do
123
+ d.feed(@time, {"a" => "b"})
124
+ end
125
+ end
126
+
90
127
  test '40x error occurred on connecting to Pub/Sub' do
91
128
  d = create_driver
92
129
 
@@ -147,6 +184,13 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
147
184
  @publisher = mock!
148
185
  @pubsub_mock = mock!.topic(anything) { @publisher }
149
186
  stub(Google::Cloud::Pubsub).new { @pubsub_mock }
187
+
188
+ @old_report_on_exception = Thread.report_on_exception
189
+ Thread.report_on_exception = false
190
+ end
191
+
192
+ teardown do
193
+ Thread.report_on_exception = @old_report_on_exception
150
194
  end
151
195
 
152
196
  test 'messages are divided into "max_messages"' do
@@ -222,5 +266,83 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
222
266
  end
223
267
  assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
224
268
  end
269
+
270
+ test 'with gzip compression' do
271
+ d = create_driver(%[
272
+ project project-test
273
+ topic topic-test
274
+ key key-test
275
+ compression gzip
276
+ ])
277
+ test_msg = {"a" => "abc"}
278
+
279
+ @publisher.publish.once
280
+ d.run(default_tag: "test") do
281
+ d.feed(test_msg)
282
+ end
283
+
284
+ assert_equal(test_msg, JSON.parse(decompress(MessagePack.unpack(d.formatted.first)[0])))
285
+ end
286
+
287
+ test 'with attribute_keys' do
288
+ d = create_driver(%[
289
+ project project-test
290
+ topic topic-test
291
+ key key-test
292
+ attribute_keys a,b
293
+ ])
294
+
295
+ @publisher.publish.once
296
+ d.run(default_tag: "test") do
297
+ d.feed(@time, {"a" => "foo", "b" => "bar", "c" => "baz"})
298
+ end
299
+
300
+ formatted = MessagePack.unpack(d.formatted[0])
301
+ message = JSON.parse(formatted[0])
302
+ assert_equal(message, {"c" => "baz"})
303
+ attributes = formatted[1]
304
+ assert_equal(attributes, {"a" => "foo", "b" => "bar"})
305
+ end
306
+
307
+ test 'with attribute_key_values' do
308
+ d = create_driver(%[
309
+ project project-test
310
+ topic topic-test
311
+ key key-test
312
+ attribute_key_values {"key1": "value1", "key2": "value2"}
313
+ ])
314
+
315
+ @publisher.publish.once
316
+ d.run(default_tag: "test") do
317
+ d.feed(@time, {"a" => "foo", "b" => "bar"})
318
+ end
319
+
320
+ formatted = MessagePack.unpack(d.formatted[0])
321
+ message = JSON.parse(formatted[0])
322
+ assert_equal(message, {"a" => "foo", "b" => "bar"})
323
+ attributes = formatted[1]
324
+ assert_equal(attributes, {"key1" => "value1", "key2" => "value2"})
325
+ end
326
+
327
+ test 'with attribute_keys and attribute_key_values' do
328
+ d = create_driver(%[
329
+ project project-test
330
+ topic topic-test
331
+ key key-test
332
+ attribute_keys a
333
+ attribute_key_values {"key": "value"}
334
+ ])
335
+
336
+ @publisher.publish.once
337
+ d.run(default_tag: "test") do
338
+ d.feed(@time, {"a" => "foo", "b" => "bar"})
339
+ end
340
+
341
+ formatted = MessagePack.unpack(d.formatted[0])
342
+ message = JSON.parse(formatted[0])
343
+ assert_equal(message, {"b" => "bar"})
344
+ attributes = formatted[1]
345
+ assert_equal(attributes, {"a" => "foo", "key" => "value"})
346
+ end
225
347
  end
226
348
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-gcloud-pubsub-custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.0
4
+ version: 1.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yoshihiro MIYAI
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-18 00:00:00.000000000 Z
11
+ date: 2021-06-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -36,14 +36,14 @@ dependencies:
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 0.30.0
39
+ version: 2.3.1
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: 0.30.0
46
+ version: 2.3.1
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: bundler
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -106,8 +106,8 @@ executables: []
106
106
  extensions: []
107
107
  extra_rdoc_files: []
108
108
  files:
109
+ - ".github/workflows/test.yml"
109
110
  - ".gitignore"
110
- - ".travis.yml"
111
111
  - CHANGELOG.md
112
112
  - Gemfile
113
113
  - LICENSE
@@ -117,14 +117,14 @@ files:
117
117
  - lib/fluent/plugin/gcloud_pubsub/client.rb
118
118
  - lib/fluent/plugin/in_gcloud_pubsub.rb
119
119
  - lib/fluent/plugin/out_gcloud_pubsub.rb
120
+ - test/helper.rb
120
121
  - test/plugin/test_in_gcloud_pubsub.rb
121
122
  - test/plugin/test_out_gcloud_pubsub.rb
122
- - test/test_helper.rb
123
123
  homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
124
124
  licenses:
125
125
  - MIT
126
126
  metadata: {}
127
- post_install_message:
127
+ post_install_message:
128
128
  rdoc_options: []
129
129
  require_paths:
130
130
  - lib
@@ -139,12 +139,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
139
139
  - !ruby/object:Gem::Version
140
140
  version: '0'
141
141
  requirements: []
142
- rubyforge_project:
143
- rubygems_version: 2.7.3
144
- signing_key:
142
+ rubygems_version: 3.1.2
143
+ signing_key:
145
144
  specification_version: 4
146
145
  summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
147
146
  test_files:
147
+ - test/helper.rb
148
148
  - test/plugin/test_in_gcloud_pubsub.rb
149
149
  - test/plugin/test_out_gcloud_pubsub.rb
150
- - test/test_helper.rb
data/.travis.yml DELETED
@@ -1,25 +0,0 @@
1
- language: ruby
2
-
3
- rvm:
4
- - 2.1
5
- - 2.2
6
- - 2.3.6
7
- - 2.4.3
8
- - 2.5.0
9
- - ruby-head
10
-
11
- gemfile:
12
- - Gemfile
13
-
14
- branches:
15
- only:
16
- - master
17
-
18
- before_install: gem update bundler
19
- script: bundle exec rake test
20
-
21
- sudo: false
22
-
23
- matrix:
24
- allow_failures:
25
- - rvm: ruby-head