fluent-plugin-formatter-protobuf 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -1
- data/CHANGELOG.md +7 -0
- data/README.md +8 -7
- data/lib/fluent/plugin/formatter_protobuf.rb +9 -1
- data/lib/fluent/plugin/version.rb +1 -1
- data/test/plugin/test_formatter_protobuf.rb +25 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1be52a5c380fb27bcda9d3773404892a45b787d81014934071dc272ad4b07249
|
|
4
|
+
data.tar.gz: ae15b5a83abe6bd2d6af1ebd41b4f47711f5f6ebda413d2b5ac799f900a140e5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d16537f2c02989d35d00edfed6db6c0fd4c4c5258ca59767a5682ab819863f25e7663a87c473fa490679bc3806de37237df0ec52e9bf918bdc05768ea0a0707f
|
|
7
|
+
data.tar.gz: 9f2d7044287ce06a11af7c91396fb1436fa695faf4537290488501fef631189662f19f27f636cc3e9f505734422a9f313aac2faf1431866aa863116322023f47
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.4.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.3.0...v0.4.0) (2022-04-11)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* Add configuration `ignore_unknown_fields` when decoding JSON ([734c638](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/commit/734c638cd2e2405c52def5babbd61fa4d69b8c08))
|
|
9
|
+
|
|
3
10
|
## [0.3.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.2.0...v0.3.0) (2022-01-13)
|
|
4
11
|
|
|
5
12
|
|
data/README.md
CHANGED
|
@@ -88,13 +88,14 @@ This plugin only supports Protobuf v3.
|
|
|
88
88
|
|
|
89
89
|
## Configuration
|
|
90
90
|
|
|
91
|
-
|parameter| type | description | default
|
|
92
|
-
|
|
93
|
-
|class_name| string (required) | Ruby Protobuf class name. Used to encode into Protobuf binary ||
|
|
94
|
-
|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary | `false`
|
|
95
|
-
|
|
|
96
|
-
|
|
|
97
|
-
|
|
|
91
|
+
| parameter | type | description | default |
|
|
92
|
+
|-----------------------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
|
|
93
|
+
| class_name | string (required) | Ruby Protobuf class name. Used to encode into Protobuf binary ||
|
|
94
|
+
| decode_json | boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary | `false` |
|
|
95
|
+
| ignore_unknown_fields | boolean (optional) | Ignore unknown fields when decoding JSON. This parameter is only used if `decode_json` is `true` | `true` |
|
|
96
|
+
| format_field | string (optional) | When defined, the plugin will only serialise the record in the given field rather than the whole record. This is potentially useful if you intend to use this formatter with the Kafka output plugin (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not wish to include those data in the resulting proto3 binary. Defaults to serialising the entire record. | `''` |
|
|
97
|
+
| include_paths | array (required) | Generated Ruby Protobuf class files path | `[]` |
|
|
98
|
+
| require_method | string (optional) | Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files are provided by a Ruby Gem, you may want to use 'require'. If you are providing the generated files as files, you may want to use 'require_relative' | `require` |
|
|
98
99
|
|
|
99
100
|
## Tips
|
|
100
101
|
|
|
@@ -41,6 +41,13 @@ module Fluent
|
|
|
41
41
|
Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary'
|
|
42
42
|
DESC
|
|
43
43
|
|
|
44
|
+
config_param :ignore_unknown_fields,
|
|
45
|
+
:bool,
|
|
46
|
+
default: true,
|
|
47
|
+
desc: <<~DESC
|
|
48
|
+
Ignore unknown fields when decoding JSON. This parameter is only used if `decode_json` is `true`
|
|
49
|
+
DESC
|
|
50
|
+
|
|
44
51
|
config_param :format_field,
|
|
45
52
|
:string,
|
|
46
53
|
default: '',
|
|
@@ -89,7 +96,8 @@ module Fluent
|
|
|
89
96
|
format_record = @format_field == '' ? record : record[@format_field]
|
|
90
97
|
|
|
91
98
|
protobuf_msg = if @decode_json
|
|
92
|
-
@protobuf_class.decode_json(Oj.dump(format_record)
|
|
99
|
+
@protobuf_class.decode_json(Oj.dump(format_record),
|
|
100
|
+
{ ignore_unknown_fields: @ignore_unknown_fields })
|
|
93
101
|
else
|
|
94
102
|
@protobuf_class.new(format_record)
|
|
95
103
|
end
|
|
@@ -83,7 +83,7 @@ class ProtobufFormatterTest < Test::Unit::TestCase
|
|
|
83
83
|
assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
-
test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true' do
|
|
86
|
+
test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true and if incoming JSON contains unknown fields' do
|
|
87
87
|
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
88
88
|
decode_json: true,
|
|
89
89
|
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
@@ -95,7 +95,8 @@ class ProtobufFormatterTest < Test::Unit::TestCase
|
|
|
95
95
|
'name' => 'Masahiro',
|
|
96
96
|
'id' => 1337,
|
|
97
97
|
'email' => 'repeatedly _at_ gmail.com',
|
|
98
|
-
'last_updated' => '2021-12-02T23:58:25.318Z'
|
|
98
|
+
'last_updated' => '2021-12-02T23:58:25.318Z',
|
|
99
|
+
'some-unknown-fields' => 'this field is not specified in the .proto message'
|
|
99
100
|
}
|
|
100
101
|
]
|
|
101
102
|
})
|
|
@@ -104,6 +105,28 @@ class ProtobufFormatterTest < Test::Unit::TestCase
|
|
|
104
105
|
assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
|
|
105
106
|
end
|
|
106
107
|
|
|
108
|
+
test 'throws exception when formatting JSON with unknown fields and ignore_unknown_fields is `false`' do
|
|
109
|
+
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
110
|
+
decode_json: true,
|
|
111
|
+
ignore_unknown_fields: false,
|
|
112
|
+
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
113
|
+
|
|
114
|
+
assert_raise(Google::Protobuf::ParseError) do
|
|
115
|
+
formatter.format('some-tag', 1234,
|
|
116
|
+
{
|
|
117
|
+
'people' => [
|
|
118
|
+
{
|
|
119
|
+
'name' => 'Masahiro',
|
|
120
|
+
'id' => 1337,
|
|
121
|
+
'email' => 'repeatedly _at_ gmail.com',
|
|
122
|
+
'last_updated' => '2021-12-02T23:58:25.318Z',
|
|
123
|
+
'some-unknown-fields' => 'this field is not specified in the .proto message'
|
|
124
|
+
}
|
|
125
|
+
]
|
|
126
|
+
})
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
107
130
|
test 'encodes Ruby hash into Protobuf binary if generated files are provided by a Gem' do
|
|
108
131
|
formatter = create_formatter({
|
|
109
132
|
class_name: 'google.protobuf.Duration',
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-formatter-protobuf
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ray Tung
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-
|
|
11
|
+
date: 2022-04-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|