fluent-plugin-gcloud-pubsub-custom 1.1.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a55a4279110490b7ac03da92af18e269d92d5d75af86d294a130acd4fb440364
4
- data.tar.gz: cfa6ba338f70ae9826fc71e3733dfb6c42175f55ae7015db38fc1703c643ed33
3
+ metadata.gz: 51c4af6bf7aee07e04161734152a796864a9a21924eab0e2434e45d177b4be0d
4
+ data.tar.gz: cd0bb6ec1e0b769b0e7792c5c8d5a797a5407ffca3dbe66f37bd1010959194ce
5
5
  SHA512:
6
- metadata.gz: 00aedffe49e51e556d3b8ef86965b67614aab97d7bb06ef1032d8b826ff8d33ca85383d822430ff8c30f6638372bd2a7f2544d86d6745b9b4f89395af384e226
7
- data.tar.gz: 0b4c66aa2418626728eebb048a4e57f1f49915d609bbf028c7979ca0ce79168e36a3d413d79301faf5ce4fb998162ae75e508b784f6764a0d08aad76a6af6d45
6
+ metadata.gz: 4f6f11c50da0f9396275e757a18cec552fdeeb8718e290a1388a2190bd26cb327704e6424848bf65a656e545f8cd6364ae571c285774cf82735a5512f0a57cd4
7
+ data.tar.gz: 62840bd5232d0f756c5ea09449f9821b2a55cee69d2fe0c048bfe764c4673fb389cde7b2b8c1b6d671a3f6def5d0d11587b080b15011fee1868079df7c35a4e8
@@ -1,11 +1,10 @@
1
1
  language: ruby
2
2
 
3
3
  rvm:
4
- - 2.1
5
- - 2.2
6
- - 2.3.6
7
- - 2.4.3
8
- - 2.5.0
4
+ - 2.4
5
+ - 2.5
6
+ - 2.6
7
+ - 2.7
9
8
  - ruby-head
10
9
 
11
10
  gemfile:
@@ -1,8 +1,35 @@
1
1
  ## ChangeLog
2
2
 
3
+ ### Release 1.4.0 - 2020/08/11
4
+
5
+ - Bump up google-cloud-pubsub to v0.39.x
6
+ - In preparation for updating to v1.x
7
+ - Output plugin
8
+ - Add `dest_project` parameter
9
+
10
+ ### Release 1.3.2 - 2019/08/16
11
+
12
+ - Input plugin
13
+ - Make `topic` parameter optional
14
+
15
+ ### Release 1.3.1 - 2018/06/08
16
+
17
+ - Input plugin
18
+ - Dumps raw string to logs for parsing errors
19
+
20
+ ### Release 1.3.0 - 2018/05/18
21
+
22
+ - Make attributes available
23
+
24
+ ### Release 1.2.0 - 2018/04/24
25
+
26
+ - Output plugin
27
+ - Support inject section
28
+
3
29
  ### Release 1.1.0 - 2018/03/30
4
30
 
5
- - The placeholder is now available in topic param
31
+ - Output plugin
32
+ - The placeholder is now available in topic param
6
33
 
7
34
  ### Release 1.0.3 - 2018/03/29
8
35
 
data/README.md CHANGED
@@ -81,6 +81,8 @@ Use `gcloud_pubsub` output plugin.
81
81
  - `topic` (required)
82
82
  - Set topic name to publish.
83
83
  - You can use placeholder in this param. See: https://docs.fluentd.org/v1.0/articles/buffer-section
84
+ - `dest_project` (optional, default: `nil`)
85
+ - Set your destination GCP project if you publish messages cross project.
84
86
  - `autocreate_topic` (optional, default: `false`)
85
87
  - If set to `true`, specified topic will be created when it doesn't exist.
86
88
  - `max_messages` (optional, default: `1000`)
@@ -91,6 +93,8 @@ Use `gcloud_pubsub` output plugin.
91
93
  - See https://cloud.google.com/pubsub/quotas#other_limits
92
94
  - `max_message_size` (optional, default: `4000000` = `4MB`)
93
95
  - Messages exceeding `max_message_size` are not published because Pub/Sub clients cannot receive it.
96
+ - `attribute_keys` (optional, default: `[]`)
97
+ - Publishing the set fields as attributes.
94
98
 
95
99
  ### Pull messages
96
100
 
@@ -131,8 +135,8 @@ Use `gcloud_pubsub` input plugin.
131
135
  - Set your credential file path.
132
136
  - Running fluentd on GCP, you can use scope instead of specifying this.
133
137
  - You can also use environment variable such as `GCLOUD_KEYFILE`.
134
- - `topic` (required)
135
- - Set topic name to pull.
138
+ - `topic` (optional)
139
+ - Set topic name that the subscription belongs to.
136
140
  - `subscription` (required)
137
141
  - Set subscription name to pull.
138
142
  - `max_messages` (optional, default: `100`)
@@ -144,6 +148,8 @@ Use `gcloud_pubsub` input plugin.
144
148
  - Pulling messages by intervals of specified seconds.
145
149
  - `pull_threads` (optional, default: `1`)
146
150
  - Set number of threads to pull messages.
151
+ - `attribute_keys` (optional, default: `[]`)
152
+ - Specify the key of the attribute to be emitted as the field of record.
147
153
  - `parse_error_action` (optional, default: `exception`)
148
154
  - Set error type when parsing messages fails.
149
155
  - `exception`: Raise exception. Messages are not acknowledged.
@@ -7,17 +7,16 @@ Gem::Specification.new do |gem|
7
7
  gem.license = "MIT"
8
8
  gem.homepage = "https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom"
9
9
  gem.summary = gem.description
10
- gem.version = "1.1.0"
10
+ gem.version = "1.4.0"
11
11
  gem.authors = ["Yoshihiro MIYAI"]
12
12
  gem.email = "msparrow17@gmail.com"
13
- gem.has_rdoc = false
14
13
  gem.files = `git ls-files`.split("\n")
15
14
  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
16
15
  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
16
  gem.require_paths = ['lib']
18
17
 
19
18
  gem.add_runtime_dependency "fluentd", [">= 0.14.15", "< 2"]
20
- gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.30.0"
19
+ gem.add_runtime_dependency "google-cloud-pubsub", "~> 0.39.0"
21
20
 
22
21
  gem.add_development_dependency "bundler"
23
22
  gem.add_development_dependency "rake"
@@ -7,19 +7,40 @@ module Fluent
7
7
  class RetryableError < Error
8
8
  end
9
9
 
10
+ class Message
11
+ attr_reader :message, :attributes
12
+ def initialize(message, attributes={})
13
+ @message = message
14
+ @attributes = attributes
15
+ end
16
+
17
+ def bytesize()
18
+ attr_size = 0
19
+ @attributes.each do |key, val|
20
+ attr_size += key.bytesize + val.bytesize
21
+ end
22
+ @message.bytesize + attr_size
23
+ end
24
+ end
25
+
10
26
  class Publisher
11
- def initialize(project, key, autocreate_topic)
27
+ def initialize(project, key, autocreate_topic, dest_project)
12
28
  @pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
13
29
  @autocreate_topic = autocreate_topic
30
+ @dest_project = dest_project
14
31
  @topics = {}
15
32
  end
16
33
 
17
34
  def topic(topic_name)
18
35
  return @topics[topic_name] if @topics.has_key? topic_name
19
36
 
20
- client = @pubsub.topic topic_name
21
- if client.nil? && @autocreate_topic
22
- client = @pubsub.create_topic topic_name
37
+ if @dest_project.nil?
38
+ client = @pubsub.topic topic_name
39
+ if client.nil? && @autocreate_topic
40
+ client = @pubsub.create_topic topic_name
41
+ end
42
+ else
43
+ client = @pubsub.topic topic_name, project: @dest_project
23
44
  end
24
45
  if client.nil?
25
46
  raise Error.new "topic:#{topic_name} does not exist."
@@ -32,7 +53,7 @@ module Fluent
32
53
  def publish(topic_name, messages)
33
54
  topic(topic_name).publish do |batch|
34
55
  messages.each do |m|
35
- batch.publish m
56
+ batch.publish m.message, m.attributes
36
57
  end
37
58
  end
38
59
  rescue Google::Cloud::UnavailableError, Google::Cloud::DeadlineExceededError, Google::Cloud::InternalError => ex
@@ -43,8 +64,12 @@ module Fluent
43
64
  class Subscriber
44
65
  def initialize(project, key, topic_name, subscription_name)
45
66
  pubsub = Google::Cloud::Pubsub.new project_id: project, credentials: key
46
- topic = pubsub.topic topic_name
47
- @client = topic.subscription subscription_name
67
+ if topic_name.nil?
68
+ @client = pubsub.subscription subscription_name
69
+ else
70
+ topic = pubsub.topic topic_name
71
+ @client = topic.subscription subscription_name
72
+ end
48
73
  raise Error.new "subscription:#{subscription_name} does not exist." if @client.nil?
49
74
  end
50
75
 
@@ -26,7 +26,7 @@ module Fluent::Plugin
26
26
  desc 'Set your credential file path.'
27
27
  config_param :key, :string, default: nil
28
28
  desc 'Set topic name to pull.'
29
- config_param :topic, :string
29
+ config_param :topic, :string, default: nil
30
30
  desc 'Set subscription name to pull.'
31
31
  config_param :subscription, :string
32
32
  desc 'Pulling messages by intervals of specified seconds.'
@@ -37,6 +37,8 @@ module Fluent::Plugin
37
37
  config_param :return_immediately, :bool, default: true
38
38
  desc 'Set number of threads to pull messages.'
39
39
  config_param :pull_threads, :integer, default: 1
40
+ desc 'Specify the key of the attribute to be acquired as a record'
41
+ config_param :attribute_keys, :array, default: []
40
42
  desc 'Set error type when parsing messages fails.'
41
43
  config_param :parse_error_action, :enum, default: :exception, list: [:exception, :warning]
42
44
  # for HTTP RPC
@@ -220,15 +222,20 @@ module Fluent::Plugin
220
222
 
221
223
  messages.each do |m|
222
224
  line = m.message.data.chomp
225
+ attributes = m.attributes
223
226
  @parser.parse(line) do |time, record|
224
227
  if time && record
228
+ @attribute_keys.each do |key|
229
+ record[key] = attributes[key]
230
+ end
231
+
225
232
  event_streams[@extract_tag.call(record)].add(time, record)
226
233
  else
227
234
  case @parse_error_action
228
235
  when :exception
229
- raise FailedParseError.new "pattern not match: #{line.inspect}"
236
+ raise FailedParseError.new "pattern not match: #{line}"
230
237
  else
231
- log.warn 'pattern not match', record: line.inspect
238
+ log.warn 'pattern not match', record: line
232
239
  end
233
240
  end
234
241
  end
@@ -1,9 +1,11 @@
1
1
  require 'fluent/plugin/output'
2
-
3
2
  require 'fluent/plugin/gcloud_pubsub/client'
3
+ require 'fluent/plugin_helper/inject'
4
4
 
5
5
  module Fluent::Plugin
6
6
  class GcloudPubSubOutput < Output
7
+ include Fluent::PluginHelper::Inject
8
+
7
9
  Fluent::Plugin.register_output('gcloud_pubsub', self)
8
10
 
9
11
  helpers :compat_parameters, :formatter
@@ -17,6 +19,8 @@ module Fluent::Plugin
17
19
  config_param :key, :string, :default => nil
18
20
  desc 'Set topic name to publish.'
19
21
  config_param :topic, :string
22
+ desc "Set your dest GCP project if publishing cross project"
23
+ config_param :dest_project, :string, :default => nil
20
24
  desc "If set to `true`, specified topic will be created when it doesn't exist."
21
25
  config_param :autocreate_topic, :bool, :default => false
22
26
  desc 'Publishing messages count per request to Cloud Pub/Sub.'
@@ -25,6 +29,8 @@ module Fluent::Plugin
25
29
  config_param :max_total_size, :integer, :default => 9800000 # 9.8MB
26
30
  desc 'Limit bytesize per message.'
27
31
  config_param :max_message_size, :integer, :default => 4000000 # 4MB
32
+ desc 'Publishing the set field as an attribute'
33
+ config_param :attribute_keys, :array, :default => []
28
34
 
29
35
  config_section :buffer do
30
36
  config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -43,11 +49,16 @@ module Fluent::Plugin
43
49
 
44
50
  def start
45
51
  super
46
- @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic
52
+ @publisher = Fluent::GcloudPubSub::Publisher.new @project, @key, @autocreate_topic, @dest_project
47
53
  end
48
54
 
49
55
  def format(tag, time, record)
50
- @formatter.format(tag, time, record).to_msgpack
56
+ record = inject_values_to_record(tag, time, record)
57
+ attributes = {}
58
+ @attribute_keys.each do |key|
59
+ attributes[key] = record.delete(key)
60
+ end
61
+ [@formatter.format(tag, time, record), attributes].to_msgpack
51
62
  end
52
63
 
53
64
  def formatted_to_msgpack_binary?
@@ -64,7 +75,8 @@ module Fluent::Plugin
64
75
  messages = []
65
76
  size = 0
66
77
 
67
- chunk.msgpack_each do |msg|
78
+ chunk.msgpack_each do |msg, attr|
79
+ msg = Fluent::GcloudPubSub::Message.new(msg, attr)
68
80
  if msg.bytesize > @max_message_size
69
81
  log.warn 'Drop a message because its size exceeds `max_message_size`', size: msg.bytesize
70
82
  next
@@ -18,13 +18,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
18
18
 
19
19
  class DummyInvalidMsgData
20
20
  def data
21
- return 'foo:bar'
21
+ 'foo:bar'
22
22
  end
23
23
  end
24
24
  class DummyInvalidMessage
25
25
  def message
26
26
  DummyInvalidMsgData.new
27
27
  end
28
+ def attributes
29
+ {"attr_1" => "a", "attr_2" => "b"}
30
+ end
28
31
  end
29
32
 
30
33
  def create_driver(conf=CONFIG)
@@ -53,6 +56,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
53
56
  return_immediately true
54
57
  pull_interval 2
55
58
  pull_threads 3
59
+ attribute_keys attr-test
56
60
  enable_rpc true
57
61
  rpc_bind 127.0.0.1
58
62
  rpc_port 24681
@@ -67,6 +71,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
67
71
  assert_equal(1000, d.instance.max_messages)
68
72
  assert_equal(true, d.instance.return_immediately)
69
73
  assert_equal(3, d.instance.pull_threads)
74
+ assert_equal(['attr-test'], d.instance.attribute_keys)
70
75
  assert_equal(true, d.instance.enable_rpc)
71
76
  assert_equal('127.0.0.1', d.instance.rpc_bind)
72
77
  assert_equal(24681, d.instance.rpc_port)
@@ -78,6 +83,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
78
83
  assert_equal(100, d.instance.max_messages)
79
84
  assert_equal(true, d.instance.return_immediately)
80
85
  assert_equal(1, d.instance.pull_threads)
86
+ assert_equal([], d.instance.attribute_keys)
81
87
  assert_equal(false, d.instance.enable_rpc)
82
88
  assert_equal('0.0.0.0', d.instance.rpc_bind)
83
89
  assert_equal(24680, d.instance.rpc_port)
@@ -126,13 +132,16 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
126
132
  sub_test_case 'emit' do
127
133
  class DummyMsgData
128
134
  def data
129
- return '{"foo": "bar"}'
135
+ '{"foo": "bar"}'
130
136
  end
131
137
  end
132
138
  class DummyMessage
133
139
  def message
134
140
  DummyMsgData.new
135
141
  end
142
+ def attributes
143
+ {"attr_1" => "a", "attr_2" => "b"}
144
+ end
136
145
  end
137
146
 
138
147
  class DummyMsgDataWithTagKey
@@ -140,7 +149,7 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
140
149
  @tag = tag
141
150
  end
142
151
  def data
143
- return '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
152
+ '{"foo": "bar", "test_tag_key": "' + @tag + '"}'
144
153
  end
145
154
  end
146
155
  class DummyMessageWithTagKey
@@ -150,6 +159,9 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
150
159
  def message
151
160
  DummyMsgDataWithTagKey.new @tag
152
161
  end
162
+ def attributes
163
+ {"attr_1" => "a", "attr_2" => "b"}
164
+ end
153
165
  end
154
166
 
155
167
  setup do
@@ -235,6 +247,22 @@ class GcloudPubSubInputTest < Test::Unit::TestCase
235
247
  assert_true d.events.empty?
236
248
  end
237
249
 
250
+ test 'with attributes' do
251
+ messages = Array.new(1, DummyMessage.new)
252
+ @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
253
+ @subscriber.acknowledge(messages).at_least(1)
254
+
255
+ d = create_driver("#{CONFIG}\nattribute_keys attr_1")
256
+ d.run(expect_emits: 1, timeout: 3)
257
+ emits = d.events
258
+
259
+ assert(1 <= emits.length)
260
+ emits.each do |tag, time, record|
261
+ assert_equal("test", tag)
262
+ assert_equal({"foo" => "bar", "attr_1" => "a"}, record)
263
+ end
264
+ end
265
+
238
266
  test 'invalid messages with parse_error_action warning' do
239
267
  messages = Array.new(1, DummyInvalidMessage.new)
240
268
  @subscriber.pull(immediate: true, max: 100).at_least(1) { messages }
@@ -62,6 +62,17 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
62
62
 
63
63
  assert_equal(true, d.instance.autocreate_topic)
64
64
  end
65
+
66
+ test '"dest_project" can be specified' do
67
+ d = create_driver(%[
68
+ project project-test
69
+ topic topic-test
70
+ key key-test
71
+ dest_project dest-project-test
72
+ ])
73
+
74
+ assert_equal("dest-project-test", d.instance.dest_project)
75
+ end
65
76
  end
66
77
 
67
78
  sub_test_case 'topic' do
@@ -87,6 +98,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
87
98
  end
88
99
  end
89
100
 
101
+ test '"dest_project" is set' do
102
+ d = create_driver(%[
103
+ project project-test
104
+ topic topic-test
105
+ key key-test
106
+ dest_project dest-project-test
107
+ ])
108
+
109
+ @publisher.publish.once
110
+ @pubsub_mock.topic("topic-test", {:project=>"dest-project-test"}).once { @publisher }
111
+ d.run(default_tag: "test") do
112
+ d.feed(@time, {"a" => "b"})
113
+ end
114
+ end
115
+
90
116
  test '40x error occurred on connecting to Pub/Sub' do
91
117
  d = create_driver
92
118
 
@@ -206,5 +232,21 @@ class GcloudPubSubOutputTest < Test::Unit::TestCase
206
232
  end
207
233
  end
208
234
  end
235
+
236
+ test 'inject section' do
237
+ d = create_driver(%[
238
+ project project-test
239
+ topic topic-test
240
+ key key-test
241
+ <inject>
242
+ tag_key tag
243
+ </inject>
244
+ ])
245
+ @publisher.publish.once
246
+ d.run(default_tag: 'test') do
247
+ d.feed({"foo" => "bar"})
248
+ end
249
+ assert_equal({"tag" => 'test', "foo" => "bar"}, JSON.parse(MessagePack.unpack(d.formatted.first)[0]))
250
+ end
209
251
  end
210
252
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-gcloud-pubsub-custom
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yoshihiro MIYAI
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-03-30 00:00:00.000000000 Z
11
+ date: 2020-08-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -36,14 +36,14 @@ dependencies:
36
36
  requirements:
37
37
  - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 0.30.0
39
+ version: 0.39.0
40
40
  type: :runtime
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
- version: 0.30.0
46
+ version: 0.39.0
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: bundler
49
49
  requirement: !ruby/object:Gem::Requirement
@@ -124,7 +124,7 @@ homepage: https://github.com/mia-0032/fluent-plugin-gcloud-pubsub-custom
124
124
  licenses:
125
125
  - MIT
126
126
  metadata: {}
127
- post_install_message:
127
+ post_install_message:
128
128
  rdoc_options: []
129
129
  require_paths:
130
130
  - lib
@@ -139,9 +139,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
139
139
  - !ruby/object:Gem::Version
140
140
  version: '0'
141
141
  requirements: []
142
- rubyforge_project:
143
- rubygems_version: 2.7.3
144
- signing_key:
142
+ rubygems_version: 3.1.2
143
+ signing_key:
145
144
  specification_version: 4
146
145
  summary: Google Cloud Pub/Sub input/output plugin for Fluentd event collector
147
146
  test_files: