fluent-plugin-formatter-protobuf 0.0.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/codeql-analysis.yml +49 -0
- data/.github/workflows/release-please.yml +48 -0
- data/.release-please-manifest.json +3 -0
- data/CHANGELOG.md +8 -0
- data/README.md +62 -17
- data/RELEASE.md +2 -7
- data/fluent-plugin-formatter-protobuf.gemspec +7 -5
- data/lib/fluent/plugin/formatter_protobuf.rb +51 -24
- data/lib/fluent/plugin/version.rb +1 -1
- data/release-please-config.json +13 -0
- data/test/integration/Gemfile +2 -2
- data/test/integration/Gemfile.lock +8 -8
- data/test/integration/fluent.conf +1 -1
- data/test/plugin/test_formatter_protobuf.rb +86 -9
- metadata +32 -17
- data/.github/workflows/publish.yml +0 -96
- data/test/proto/addressbook +0 -1
- data/test/proto/addressbook.bin +0 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e6218edd36cd2e7186b2def09700237885d5851931a15ad3859b8eecdc8fd1ca
|
|
4
|
+
data.tar.gz: f609d96e5a3dd326ee0cc5d9a9638942ed9f9f063f34bca7ad696176ed58f589
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: de97c9eed15e2ff1f86a4042412ad59efedad063a6957f052828b5efd5a3e6c888e9a1aa3f2f0c4cc1b1c33b3906a48d91dbd4d1b8dd855ece6c24db2be2742d
|
|
7
|
+
data.tar.gz: 45365b906912baf789221d565b039a44a4c97ac596518da587778ece2e92856b027a6ac5768cfae8fd64c9dd5b4032195ba2fc874c73cb1b804b9c93286c7682
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: "CodeQL"
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [ main ]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [ main ]
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: '18 11 * * 4'
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
analyze:
|
|
13
|
+
name: Analyze
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
permissions:
|
|
16
|
+
actions: read
|
|
17
|
+
contents: read
|
|
18
|
+
security-events: write
|
|
19
|
+
|
|
20
|
+
strategy:
|
|
21
|
+
fail-fast: false
|
|
22
|
+
matrix:
|
|
23
|
+
language: [ 'ruby' ]
|
|
24
|
+
|
|
25
|
+
steps:
|
|
26
|
+
- name: Checkout repository
|
|
27
|
+
uses: actions/checkout@v2
|
|
28
|
+
|
|
29
|
+
- name: Initialize CodeQL
|
|
30
|
+
uses: github/codeql-action/init@v1
|
|
31
|
+
with:
|
|
32
|
+
languages: ${{ matrix.language }}
|
|
33
|
+
|
|
34
|
+
- name: Autobuild
|
|
35
|
+
uses: github/codeql-action/autobuild@v1
|
|
36
|
+
|
|
37
|
+
# ℹ️ Command-line programs to run using the OS shell.
|
|
38
|
+
# 📚 https://git.io/JvXDl
|
|
39
|
+
|
|
40
|
+
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
|
|
41
|
+
# and modify them (or add more) to build your code if your project
|
|
42
|
+
# uses a compiled language
|
|
43
|
+
|
|
44
|
+
#- run: |
|
|
45
|
+
# make bootstrap
|
|
46
|
+
# make release
|
|
47
|
+
|
|
48
|
+
- name: Perform CodeQL Analysis
|
|
49
|
+
uses: github/codeql-action/analyze@v1
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
on:
|
|
2
|
+
push:
|
|
3
|
+
branches:
|
|
4
|
+
- main
|
|
5
|
+
name: release-please
|
|
6
|
+
jobs:
|
|
7
|
+
release-please:
|
|
8
|
+
environment: main
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: google-github-actions/release-please-action@v2
|
|
12
|
+
id: release
|
|
13
|
+
with:
|
|
14
|
+
token: ${{ secrets.GITHUB_TOKEN }}
|
|
15
|
+
release-type: ruby
|
|
16
|
+
version-file: "lib/fluent/plugin/version.rb"
|
|
17
|
+
|
|
18
|
+
- uses: actions/checkout@v2
|
|
19
|
+
|
|
20
|
+
- name: Set up Ruby 2.5
|
|
21
|
+
uses: ruby/setup-ruby@v1
|
|
22
|
+
with:
|
|
23
|
+
bundler-cache: true
|
|
24
|
+
|
|
25
|
+
- name: Publish to GPR
|
|
26
|
+
run: |
|
|
27
|
+
mkdir -p $HOME/.gem
|
|
28
|
+
touch $HOME/.gem/credentials
|
|
29
|
+
chmod 0600 $HOME/.gem/credentials
|
|
30
|
+
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
31
|
+
gem build *.gemspec
|
|
32
|
+
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
|
|
33
|
+
env:
|
|
34
|
+
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
|
35
|
+
OWNER: ${{ github.repository_owner }}
|
|
36
|
+
if: ${{ steps.release.outputs.release_created }}
|
|
37
|
+
|
|
38
|
+
- name: Publish to RubyGems
|
|
39
|
+
run: |
|
|
40
|
+
mkdir -p $HOME/.gem
|
|
41
|
+
touch $HOME/.gem/credentials
|
|
42
|
+
chmod 0600 $HOME/.gem/credentials
|
|
43
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
44
|
+
gem build *.gemspec
|
|
45
|
+
gem push *.gem
|
|
46
|
+
env:
|
|
47
|
+
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
|
48
|
+
if: ${{ steps.release.outputs.release_created }}
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.3.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.2.0...v0.3.0) (2022-01-13)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* Add new configuration - `format_field` ([797789b](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/commit/797789b38d5a599c2c107979beb7edf66e384bd5))
|
data/README.md
CHANGED
|
@@ -18,7 +18,7 @@ $ gem install fluent-plugin-formatter-protobuf
|
|
|
18
18
|
$ gem install \
|
|
19
19
|
fluent-plugin-formatter-protobuf \
|
|
20
20
|
--version "<version>" \
|
|
21
|
-
--source "https://rubygems.pkg.github.com/
|
|
21
|
+
--source "https://rubygems.pkg.github.com/fluent-plugins-nursery"
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
### Bundler (RubyGems)
|
|
@@ -32,7 +32,7 @@ gem "fluent-plugin-formatter-protobuf", "<version>"
|
|
|
32
32
|
### Bundler (GitHub RubyGems Registry),
|
|
33
33
|
|
|
34
34
|
```shell
|
|
35
|
-
source "https://rubygems.pkg.github.com/
|
|
35
|
+
source "https://rubygems.pkg.github.com/fluent-plugins-nursery" do
|
|
36
36
|
gem "fluent-plugin-formatter-protobuf", "<version>"
|
|
37
37
|
end
|
|
38
38
|
```
|
|
@@ -43,16 +43,20 @@ And then execute:
|
|
|
43
43
|
$ bundle
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
##
|
|
46
|
+
## Quick start
|
|
47
47
|
|
|
48
48
|
1. Generate the protobuf ruby methods
|
|
49
49
|
1. e.g `protoc --proto_path=. --ruby_out=. ./log.proto`
|
|
50
|
-
2. Modify the `<format>` section to include `
|
|
50
|
+
2. Modify the `<format>` section to include `class_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
|
|
51
|
+
1. Given protobuf class `Your::Protobuf::Class::Name` class should be given as `Your.Protobuf.Class.Name` in `class_name`. The exact name can be found in the generated Ruby files
|
|
51
52
|
|
|
53
|
+
## Note
|
|
54
|
+
|
|
55
|
+
This plugin only supports Protobuf v3.
|
|
52
56
|
|
|
53
57
|
## Example
|
|
54
58
|
|
|
55
|
-
```
|
|
59
|
+
```aconf
|
|
56
60
|
<source>
|
|
57
61
|
@type tcp
|
|
58
62
|
tag tcp.events
|
|
@@ -64,17 +68,19 @@ $ bundle
|
|
|
64
68
|
</source>
|
|
65
69
|
|
|
66
70
|
<match tcp.events>
|
|
67
|
-
@type
|
|
68
|
-
|
|
69
|
-
|
|
71
|
+
@type kafka2
|
|
72
|
+
|
|
73
|
+
brokers "#{BROKER_ENDPOINTS}"
|
|
74
|
+
compression_codec lz4
|
|
70
75
|
|
|
71
|
-
<buffer>
|
|
76
|
+
<buffer topic>
|
|
72
77
|
@type memory
|
|
78
|
+
flush_interval 5s
|
|
73
79
|
</buffer>
|
|
74
80
|
|
|
75
81
|
<format>
|
|
76
82
|
@type protobuf
|
|
77
|
-
|
|
83
|
+
class_name "Log"
|
|
78
84
|
include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
|
|
79
85
|
</format>
|
|
80
86
|
</match>
|
|
@@ -82,16 +88,55 @@ $ bundle
|
|
|
82
88
|
|
|
83
89
|
## Configuration
|
|
84
90
|
|
|
85
|
-
|
|
91
|
+
|parameter| type | description | default |
|
|
92
|
+
|---|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|
|
|
93
|
+
|class_name| string (required) | Ruby Protobuf class name. Used to encode into Protobuf binary ||
|
|
94
|
+
|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary | `false` |
|
|
95
|
+
|format_field| string (optional) | When defined, the plugin will only serialise the record in the given field rather than the whole record. This is potentially useful if you intend to use this formatter with the Kafka output plugin (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not wish to include those data in the resulting proto3 binary. Defaults to serialising the entire record. | `''` |
|
|
96
|
+
|include_paths| array (required) | Generated Ruby Protobuf class files path | `[]` |
|
|
97
|
+
|require_method| string (optional) | Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files are provided by a Ruby Gem, you may want to use 'require'. If you are providing the generated files as files, you may want to use 'require_relative' | `require` |
|
|
98
|
+
|
|
99
|
+
## Tips
|
|
100
|
+
|
|
101
|
+
#### 1. I have multiple `_pb.rb` files and they depend on one another. When I use this plugin, I'm getting a `LoadError`.
|
|
102
|
+
|
|
103
|
+
This happens because the plugin currently loads paths in `include_paths` sequentially. You can either sort the files in correct dependency order (which is cumbersome), or add to Ruby's `$LOAD_PATH`.
|
|
104
|
+
|
|
105
|
+
For example, you have 2 generated ruby files, `a_pb.rb` and `b_pb.rb`, and both `a_pb.rb` depends on `b_pb.rb`.
|
|
106
|
+
|
|
107
|
+
You can either order them in appropriate dependency order, such as
|
|
108
|
+
|
|
109
|
+
```aconf
|
|
110
|
+
<format>
|
|
111
|
+
@type protobuf
|
|
112
|
+
class_name "SomePackage.A"
|
|
113
|
+
include_paths ["/some/path/b_pb.rb", "/some/path/a_pb.rb"]
|
|
114
|
+
</format>
|
|
86
115
|
|
|
87
116
|
```
|
|
88
|
-
|
|
117
|
+
|
|
118
|
+
or you can put the generated files in a directory say `/some/path/`, and add to Ruby's `$LOAD_PATH` via the `RUBYLIB` environment variable.
|
|
119
|
+
|
|
120
|
+
e.g.
|
|
121
|
+
|
|
122
|
+
```aconf
|
|
123
|
+
<format>
|
|
124
|
+
@type protobuf
|
|
125
|
+
class_name "SomePackage.A"
|
|
126
|
+
include_paths ["/some/path/a_pb.rb"]
|
|
127
|
+
</format>
|
|
128
|
+
|
|
89
129
|
```
|
|
90
130
|
|
|
91
|
-
|
|
131
|
+
```shell
|
|
132
|
+
|
|
133
|
+
export RUBYLIB="${RUBYLIB}:/some/path/"
|
|
134
|
+
fluentd \
|
|
135
|
+
--gemfile "/some/other/path/Gemfile" \
|
|
136
|
+
--config "/some/other/path/fluent.conf"
|
|
137
|
+
|
|
138
|
+
```
|
|
92
139
|
|
|
93
|
-
##
|
|
140
|
+
## License
|
|
94
141
|
|
|
95
|
-
|
|
96
|
-
* License
|
|
97
|
-
* Apache License, Version 2.0
|
|
142
|
+
[Apache License, Version 2.0](./LICENSE)
|
data/RELEASE.md
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
1
|
# Release
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
1. Update `lib/fluent/version.rb`
|
|
7
|
-
2. Update `Gemfile.lock`
|
|
8
|
-
3. Create a pull request and merge it into `main`
|
|
9
|
-
4. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`
|
|
3
|
+
This repository follow conventional commit and semantic versioning, and the release to RubyGems.org and GitHub Ruby
|
|
4
|
+
Repository is managed by [release-please](https://github.com/google-github-actions/release-please-action).
|
|
@@ -9,6 +9,7 @@ Gem::Specification.new do |spec|
|
|
|
9
9
|
spec.version = Fluent::Plugin::VERSION
|
|
10
10
|
spec.authors = ['Ray Tung']
|
|
11
11
|
spec.email = ['code@raytung.net']
|
|
12
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
|
12
13
|
|
|
13
14
|
spec.summary = 'Protobuf formatter for Fluentd'
|
|
14
15
|
spec.description = 'This is a Fluentd formatter plugin designed to convert Protobuf JSON into Protobuf binary'
|
|
@@ -28,11 +29,12 @@ Gem::Specification.new do |spec|
|
|
|
28
29
|
spec.test_files = test_files
|
|
29
30
|
spec.require_paths = ['lib']
|
|
30
31
|
|
|
31
|
-
spec.add_development_dependency 'bundler', '~> 2.2
|
|
32
|
-
spec.add_development_dependency 'rake', '~> 13.0
|
|
33
|
-
spec.add_development_dependency 'rubocop', '~> 1.22
|
|
34
|
-
spec.add_development_dependency 'rubocop-rake', '~> 0.6
|
|
35
|
-
spec.add_development_dependency 'test-unit', '~> 3.3
|
|
32
|
+
spec.add_development_dependency 'bundler', '~> 2.2'
|
|
33
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
|
34
|
+
spec.add_development_dependency 'rubocop', '~> 1.22'
|
|
35
|
+
spec.add_development_dependency 'rubocop-rake', '~> 0.6'
|
|
36
|
+
spec.add_development_dependency 'test-unit', '~> 3.3'
|
|
36
37
|
spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
|
|
37
38
|
spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']
|
|
39
|
+
spec.add_runtime_dependency 'oj', '~> 3.13'
|
|
38
40
|
end
|
|
@@ -22,6 +22,7 @@ require 'google/protobuf'
|
|
|
22
22
|
|
|
23
23
|
require 'fluent/env'
|
|
24
24
|
require 'fluent/time'
|
|
25
|
+
require 'oj'
|
|
25
26
|
|
|
26
27
|
module Fluent
|
|
27
28
|
module Plugin
|
|
@@ -29,11 +30,43 @@ module Fluent
|
|
|
29
30
|
class ProtobufFormatter < Fluent::Plugin::Formatter
|
|
30
31
|
Fluent::Plugin.register_formatter('protobuf', self)
|
|
31
32
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
config_param :
|
|
33
|
+
config_param :class_name,
|
|
34
|
+
:string,
|
|
35
|
+
desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
|
|
36
|
+
|
|
37
|
+
config_param :decode_json,
|
|
38
|
+
:bool,
|
|
39
|
+
default: false,
|
|
40
|
+
desc: <<~DESC
|
|
41
|
+
Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary'
|
|
42
|
+
DESC
|
|
43
|
+
|
|
44
|
+
config_param :format_field,
|
|
45
|
+
:string,
|
|
46
|
+
default: '',
|
|
47
|
+
desc: <<~DESC
|
|
48
|
+
When defined, the plugin will only serialise the record in the given field rather than the whole record.
|
|
49
|
+
This is potentially useful if you intend to use this formatter with the Kafka output plugin
|
|
50
|
+
(https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains
|
|
51
|
+
a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not
|
|
52
|
+
wish to include those data in the resulting proto3 binary.
|
|
53
|
+
|
|
54
|
+
Defaults to serialising the whole record.
|
|
55
|
+
DESC
|
|
56
|
+
|
|
57
|
+
config_param :include_paths,
|
|
58
|
+
:array,
|
|
59
|
+
default: [],
|
|
60
|
+
desc: 'Generated Ruby Protobuf class files path'
|
|
61
|
+
|
|
62
|
+
config_param :require_method,
|
|
63
|
+
:string,
|
|
64
|
+
default: 'require',
|
|
65
|
+
desc: <<~DESC
|
|
66
|
+
Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files
|
|
67
|
+
are provided by a Ruby Gem, you would want to use \'require\'. If you are providing the generated files as files, you
|
|
68
|
+
may want to use \'require_relative\''
|
|
69
|
+
DESC
|
|
37
70
|
|
|
38
71
|
def configure(conf)
|
|
39
72
|
super(conf)
|
|
@@ -42,10 +75,10 @@ module Fluent
|
|
|
42
75
|
|
|
43
76
|
@include_paths.each { |path| require_proto!(path) } unless @include_paths.empty?
|
|
44
77
|
|
|
45
|
-
|
|
46
|
-
raise Fluent::ConfigError, "
|
|
78
|
+
class_lookup = Google::Protobuf::DescriptorPool.generated_pool.lookup(@class_name)
|
|
79
|
+
raise Fluent::ConfigError, "class name '#{@class_name}' not found" if class_lookup.nil?
|
|
47
80
|
|
|
48
|
-
@protobuf_class =
|
|
81
|
+
@protobuf_class = class_lookup.msgclass
|
|
49
82
|
end
|
|
50
83
|
|
|
51
84
|
def formatter_type
|
|
@@ -53,26 +86,20 @@ module Fluent
|
|
|
53
86
|
end
|
|
54
87
|
|
|
55
88
|
def format(_tag, _time, record)
|
|
56
|
-
|
|
89
|
+
format_record = @format_field == '' ? record : record[@format_field]
|
|
90
|
+
|
|
91
|
+
protobuf_msg = if @decode_json
|
|
92
|
+
@protobuf_class.decode_json(Oj.dump(format_record))
|
|
93
|
+
else
|
|
94
|
+
@protobuf_class.new(format_record)
|
|
95
|
+
end
|
|
57
96
|
@protobuf_class.encode(protobuf_msg)
|
|
58
97
|
end
|
|
59
98
|
|
|
60
|
-
# @param [string] filename
|
|
61
|
-
# @return [void]
|
|
62
99
|
def require_proto!(filename)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
unless Pathname.new(filename).absolute?
|
|
68
|
-
raise Fluent::ConfigError, "Unable to load file '#{filename}'. Please provide absolute paths"
|
|
69
|
-
end
|
|
70
|
-
|
|
71
|
-
begin
|
|
72
|
-
require filename
|
|
73
|
-
rescue LoadError => e
|
|
74
|
-
raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.inspect}"
|
|
75
|
-
end
|
|
100
|
+
Kernel.method(@require_method.to_sym).call(filename)
|
|
101
|
+
rescue LoadError => e
|
|
102
|
+
raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.message}"
|
|
76
103
|
end
|
|
77
104
|
end
|
|
78
105
|
end
|
data/test/integration/Gemfile
CHANGED
|
@@ -3,23 +3,23 @@ GEM
|
|
|
3
3
|
specs:
|
|
4
4
|
concurrent-ruby (1.1.9)
|
|
5
5
|
cool.io (1.7.1)
|
|
6
|
-
fluent-plugin-formatter-protobuf (0.0
|
|
6
|
+
fluent-plugin-formatter-protobuf (0.1.0)
|
|
7
7
|
fluentd (>= 1.0, < 2)
|
|
8
8
|
google-protobuf (~> 3.18)
|
|
9
|
-
fluentd (1.14.
|
|
9
|
+
fluentd (1.14.2)
|
|
10
10
|
bundler
|
|
11
11
|
cool.io (>= 1.4.5, < 2.0.0)
|
|
12
12
|
http_parser.rb (>= 0.5.1, < 0.8.0)
|
|
13
13
|
msgpack (>= 1.3.1, < 2.0.0)
|
|
14
14
|
serverengine (>= 2.2.2, < 3.0.0)
|
|
15
15
|
sigdump (~> 0.2.2)
|
|
16
|
-
strptime (>= 0.2.
|
|
16
|
+
strptime (>= 0.2.4, < 1.0.0)
|
|
17
17
|
tzinfo (>= 1.0, < 3.0)
|
|
18
18
|
tzinfo-data (~> 1.0)
|
|
19
19
|
webrick (>= 1.4.2, < 1.8.0)
|
|
20
20
|
yajl-ruby (~> 1.0)
|
|
21
|
-
google-protobuf (3.
|
|
22
|
-
google-protobuf (3.
|
|
21
|
+
google-protobuf (3.19.2)
|
|
22
|
+
google-protobuf (3.19.2-x86_64-linux)
|
|
23
23
|
http_parser.rb (0.7.0)
|
|
24
24
|
msgpack (1.4.2)
|
|
25
25
|
serverengine (2.2.4)
|
|
@@ -28,7 +28,7 @@ GEM
|
|
|
28
28
|
strptime (0.2.5)
|
|
29
29
|
tzinfo (2.0.4)
|
|
30
30
|
concurrent-ruby (~> 1.0)
|
|
31
|
-
tzinfo-data (1.2021.
|
|
31
|
+
tzinfo-data (1.2021.5)
|
|
32
32
|
tzinfo (>= 1.0.0)
|
|
33
33
|
webrick (1.7.0)
|
|
34
34
|
yajl-ruby (1.4.1)
|
|
@@ -38,8 +38,8 @@ PLATFORMS
|
|
|
38
38
|
x86_64-linux
|
|
39
39
|
|
|
40
40
|
DEPENDENCIES
|
|
41
|
-
fluent-plugin-formatter-protobuf (= 0.0
|
|
42
|
-
fluentd (= 1.14.
|
|
41
|
+
fluent-plugin-formatter-protobuf (= 0.1.0)
|
|
42
|
+
fluentd (= 1.14.2)
|
|
43
43
|
|
|
44
44
|
BUNDLED WITH
|
|
45
45
|
2.2.29
|
|
@@ -8,38 +8,115 @@ class ProtobufFormatterTest < Test::Unit::TestCase
|
|
|
8
8
|
Fluent::Test.setup
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
VALID_INCLUDE_PATHS_ABSOLUTE = [File.expand_path(File.join(__dir__, '..', 'proto', 'addressbook_pb.rb'))].freeze
|
|
12
12
|
|
|
13
|
+
# Relative to the plugin file
|
|
14
|
+
VALID_INCLUDE_PATHS_RELATIVE = '../../../test/proto/addressbook_pb.rb'
|
|
15
|
+
|
|
16
|
+
# rubocop:disable Metrics/BlockLength
|
|
13
17
|
sub_test_case 'configure' do
|
|
14
18
|
test 'fail if include_paths is empty' do
|
|
15
19
|
assert_raise(Fluent::ConfigError) do
|
|
16
|
-
create_driver({
|
|
20
|
+
create_driver({ class_name: '', include_paths: [] })
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
test 'fail if ruby files not found in the provided include paths' do
|
|
25
|
+
assert_raise(Fluent::ConfigError) do
|
|
26
|
+
create_driver({ class_name: 'tutorial.AddressBook', include_paths: ['some/random/path'] })
|
|
17
27
|
end
|
|
18
28
|
end
|
|
19
29
|
|
|
20
|
-
test 'fail if no protobuf class can be found with
|
|
30
|
+
test 'fail if no protobuf class can be found with class_name' do
|
|
21
31
|
assert_raise(Fluent::ConfigError) do
|
|
22
|
-
create_driver({
|
|
32
|
+
create_driver({ class_name: 'Some.Name', include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
test 'success if given valid relative paths in include paths' do
|
|
37
|
+
assert_nothing_raised do
|
|
38
|
+
create_driver({
|
|
39
|
+
class_name: 'tutorial.AddressBook',
|
|
40
|
+
include_paths: [VALID_INCLUDE_PATHS_RELATIVE],
|
|
41
|
+
require_method: 'require_relative'
|
|
42
|
+
})
|
|
23
43
|
end
|
|
24
44
|
end
|
|
25
45
|
|
|
26
46
|
test 'passes on valid configuration' do
|
|
27
47
|
assert_nothing_raised do
|
|
28
|
-
create_driver({
|
|
48
|
+
create_driver({ class_name: 'tutorial.AddressBook', include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
29
49
|
end
|
|
30
50
|
end
|
|
31
51
|
end
|
|
52
|
+
# rubocop:enable Metrics/BlockLength
|
|
32
53
|
|
|
54
|
+
stub_ruby_hash = { 'people' => [{ 'name' => 'Masahiro', 'id' => 1337,
|
|
55
|
+
'email' => 'repeatedly _at_ gmail.com',
|
|
56
|
+
'last_updated' => {
|
|
57
|
+
'seconds' => 1_638_489_505,
|
|
58
|
+
'nanos' => 318_000_000
|
|
59
|
+
} }] }
|
|
60
|
+
# rubocop:disable Metrics/BlockLength
|
|
33
61
|
sub_test_case 'format' do
|
|
34
62
|
test 'encodes into Protobuf binary' do
|
|
35
|
-
formatter = create_formatter({
|
|
63
|
+
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
64
|
+
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
65
|
+
|
|
66
|
+
formatted = formatter.format('some-tag', 1234, stub_ruby_hash)
|
|
67
|
+
address_book = Tutorial::AddressBook.new(stub_ruby_hash)
|
|
68
|
+
assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
test 'encodes a particular field instead of the entire record if format_field is defined' do
|
|
72
|
+
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
73
|
+
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE,
|
|
74
|
+
format_field: 'data' })
|
|
36
75
|
|
|
37
76
|
formatted = formatter.format('some-tag', 1234,
|
|
38
|
-
{
|
|
39
|
-
|
|
40
|
-
|
|
77
|
+
{
|
|
78
|
+
'topic' => 'some-kafka-topic',
|
|
79
|
+
'headers' => {},
|
|
80
|
+
'data' => stub_ruby_hash
|
|
81
|
+
})
|
|
82
|
+
address_book = Tutorial::AddressBook.new(stub_ruby_hash)
|
|
83
|
+
assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true' do
|
|
87
|
+
formatter = create_formatter({ class_name: 'tutorial.AddressBook',
|
|
88
|
+
decode_json: true,
|
|
89
|
+
include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
|
|
90
|
+
|
|
91
|
+
formatted = formatter.format('some-tag', 1234,
|
|
92
|
+
{
|
|
93
|
+
'people' => [
|
|
94
|
+
{
|
|
95
|
+
'name' => 'Masahiro',
|
|
96
|
+
'id' => 1337,
|
|
97
|
+
'email' => 'repeatedly _at_ gmail.com',
|
|
98
|
+
'last_updated' => '2021-12-02T23:58:25.318Z'
|
|
99
|
+
}
|
|
100
|
+
]
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
address_book = Tutorial::AddressBook.new(stub_ruby_hash)
|
|
104
|
+
assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
test 'encodes Ruby hash into Protobuf binary if generated files are provided by a Gem' do
|
|
108
|
+
formatter = create_formatter({
|
|
109
|
+
class_name: 'google.protobuf.Duration',
|
|
110
|
+
# Provided by the google-protobuf gem
|
|
111
|
+
include_paths: ['google/protobuf/duration_pb'],
|
|
112
|
+
require_method: 'require'
|
|
113
|
+
})
|
|
114
|
+
formatted = formatter.format('some-tag', 1234, { seconds: 1, nanos: 340_012 })
|
|
115
|
+
duration = Google::Protobuf::Duration.new({ seconds: 1, nanos: 340_012 })
|
|
116
|
+
assert_equal(Google::Protobuf::Duration.encode(duration), formatted)
|
|
41
117
|
end
|
|
42
118
|
end
|
|
119
|
+
# rubocop:enable Metrics/BlockLength
|
|
43
120
|
|
|
44
121
|
private
|
|
45
122
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-formatter-protobuf
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ray Tung
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-01-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -16,70 +16,70 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: 2.2
|
|
19
|
+
version: '2.2'
|
|
20
20
|
type: :development
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: 2.2
|
|
26
|
+
version: '2.2'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: rake
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
31
|
- - "~>"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: 13.0
|
|
33
|
+
version: '13.0'
|
|
34
34
|
type: :development
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: 13.0
|
|
40
|
+
version: '13.0'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
42
|
name: rubocop
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - "~>"
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: 1.22
|
|
47
|
+
version: '1.22'
|
|
48
48
|
type: :development
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
52
|
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: 1.22
|
|
54
|
+
version: '1.22'
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: rubocop-rake
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
59
|
- - "~>"
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: 0.6
|
|
61
|
+
version: '0.6'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
66
|
- - "~>"
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: 0.6
|
|
68
|
+
version: '0.6'
|
|
69
69
|
- !ruby/object:Gem::Dependency
|
|
70
70
|
name: test-unit
|
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
|
72
72
|
requirements:
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: 3.3
|
|
75
|
+
version: '3.3'
|
|
76
76
|
type: :development
|
|
77
77
|
prerelease: false
|
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
79
|
requirements:
|
|
80
80
|
- - "~>"
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: 3.3
|
|
82
|
+
version: '3.3'
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
|
84
84
|
name: fluentd
|
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -114,6 +114,20 @@ dependencies:
|
|
|
114
114
|
- - "~>"
|
|
115
115
|
- !ruby/object:Gem::Version
|
|
116
116
|
version: '3.18'
|
|
117
|
+
- !ruby/object:Gem::Dependency
|
|
118
|
+
name: oj
|
|
119
|
+
requirement: !ruby/object:Gem::Requirement
|
|
120
|
+
requirements:
|
|
121
|
+
- - "~>"
|
|
122
|
+
- !ruby/object:Gem::Version
|
|
123
|
+
version: '3.13'
|
|
124
|
+
type: :runtime
|
|
125
|
+
prerelease: false
|
|
126
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
127
|
+
requirements:
|
|
128
|
+
- - "~>"
|
|
129
|
+
- !ruby/object:Gem::Version
|
|
130
|
+
version: '3.13'
|
|
117
131
|
description: This is a Fluentd formatter plugin designed to convert Protobuf JSON
|
|
118
132
|
into Protobuf binary
|
|
119
133
|
email:
|
|
@@ -123,10 +137,13 @@ extensions: []
|
|
|
123
137
|
extra_rdoc_files: []
|
|
124
138
|
files:
|
|
125
139
|
- ".github/workflows/build-and-test.yml"
|
|
126
|
-
- ".github/workflows/
|
|
140
|
+
- ".github/workflows/codeql-analysis.yml"
|
|
141
|
+
- ".github/workflows/release-please.yml"
|
|
127
142
|
- ".gitignore"
|
|
143
|
+
- ".release-please-manifest.json"
|
|
128
144
|
- ".rubocop.yml"
|
|
129
145
|
- ".ruby-version"
|
|
146
|
+
- CHANGELOG.md
|
|
130
147
|
- Gemfile
|
|
131
148
|
- LICENSE
|
|
132
149
|
- README.md
|
|
@@ -137,6 +154,7 @@ files:
|
|
|
137
154
|
- fluent-plugin-formatter-protobuf.gemspec
|
|
138
155
|
- lib/fluent/plugin/formatter_protobuf.rb
|
|
139
156
|
- lib/fluent/plugin/version.rb
|
|
157
|
+
- release-please-config.json
|
|
140
158
|
- test/helper.rb
|
|
141
159
|
- test/integration/Gemfile
|
|
142
160
|
- test/integration/Gemfile.lock
|
|
@@ -147,14 +165,13 @@ files:
|
|
|
147
165
|
- test/integration/run.sh
|
|
148
166
|
- test/plugin/test_formatter_protobuf.rb
|
|
149
167
|
- test/proto/README.md
|
|
150
|
-
- test/proto/addressbook
|
|
151
|
-
- test/proto/addressbook.bin
|
|
152
168
|
- test/proto/addressbook.proto
|
|
153
169
|
- test/proto/addressbook_pb.rb
|
|
154
170
|
homepage: https://github.com/raytung/fluent-plugin-formatter-protobuf
|
|
155
171
|
licenses:
|
|
156
172
|
- Apache-2.0
|
|
157
173
|
metadata:
|
|
174
|
+
rubygems_mfa_required: 'true'
|
|
158
175
|
homepage_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
|
|
159
176
|
source_code_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
|
|
160
177
|
changelog_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf/blob/main/CHANGELOG.md
|
|
@@ -189,7 +206,5 @@ test_files:
|
|
|
189
206
|
- test/integration/run.sh
|
|
190
207
|
- test/plugin/test_formatter_protobuf.rb
|
|
191
208
|
- test/proto/README.md
|
|
192
|
-
- test/proto/addressbook
|
|
193
|
-
- test/proto/addressbook.bin
|
|
194
209
|
- test/proto/addressbook.proto
|
|
195
210
|
- test/proto/addressbook_pb.rb
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
name: Publish
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
tags:
|
|
6
|
-
- v0.*
|
|
7
|
-
|
|
8
|
-
jobs:
|
|
9
|
-
lint:
|
|
10
|
-
name: Lint
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
permissions:
|
|
13
|
-
packages: write
|
|
14
|
-
contents: read
|
|
15
|
-
|
|
16
|
-
steps:
|
|
17
|
-
- uses: actions/checkout@v2
|
|
18
|
-
- name: Set up Ruby 2.5
|
|
19
|
-
uses: ruby/setup-ruby@v1
|
|
20
|
-
with:
|
|
21
|
-
bundler-cache: true
|
|
22
|
-
- name: "Unit test"
|
|
23
|
-
run: "bundle exec rake lint:check"
|
|
24
|
-
|
|
25
|
-
test:
|
|
26
|
-
name: Unit test
|
|
27
|
-
runs-on: ubuntu-latest
|
|
28
|
-
permissions:
|
|
29
|
-
packages: write
|
|
30
|
-
contents: read
|
|
31
|
-
|
|
32
|
-
steps:
|
|
33
|
-
- uses: actions/checkout@v2
|
|
34
|
-
- name: Set up Ruby 2.5
|
|
35
|
-
uses: ruby/setup-ruby@v1
|
|
36
|
-
with:
|
|
37
|
-
bundler-cache: true
|
|
38
|
-
- name: "Unit test"
|
|
39
|
-
run: "bundle exec rake test:unit"
|
|
40
|
-
|
|
41
|
-
publish-to-gpr:
|
|
42
|
-
needs:
|
|
43
|
-
- lint
|
|
44
|
-
- test
|
|
45
|
-
name: Build + Publish to GPR
|
|
46
|
-
runs-on: ubuntu-latest
|
|
47
|
-
permissions:
|
|
48
|
-
packages: write
|
|
49
|
-
contents: read
|
|
50
|
-
|
|
51
|
-
steps:
|
|
52
|
-
- uses: actions/checkout@v2
|
|
53
|
-
- name: Set up Ruby 2.5
|
|
54
|
-
uses: ruby/setup-ruby@v1
|
|
55
|
-
with:
|
|
56
|
-
bundler-cache: true
|
|
57
|
-
|
|
58
|
-
- name: Publish to GPR
|
|
59
|
-
run: |
|
|
60
|
-
mkdir -p $HOME/.gem
|
|
61
|
-
touch $HOME/.gem/credentials
|
|
62
|
-
chmod 0600 $HOME/.gem/credentials
|
|
63
|
-
printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
64
|
-
gem build *.gemspec
|
|
65
|
-
gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
|
|
66
|
-
env:
|
|
67
|
-
GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
|
|
68
|
-
OWNER: ${{ github.repository_owner }}
|
|
69
|
-
|
|
70
|
-
publish-to-ruby-gems:
|
|
71
|
-
needs:
|
|
72
|
-
- lint
|
|
73
|
-
- test
|
|
74
|
-
name: Build + Publish to RubyGems
|
|
75
|
-
runs-on: ubuntu-latest
|
|
76
|
-
permissions:
|
|
77
|
-
packages: write
|
|
78
|
-
contents: read
|
|
79
|
-
|
|
80
|
-
steps:
|
|
81
|
-
- uses: actions/checkout@v2
|
|
82
|
-
- name: Set up Ruby 2.5
|
|
83
|
-
uses: ruby/setup-ruby@v1
|
|
84
|
-
with:
|
|
85
|
-
bundler-cache: true
|
|
86
|
-
|
|
87
|
-
- name: Publish to RubyGems
|
|
88
|
-
run: |
|
|
89
|
-
mkdir -p $HOME/.gem
|
|
90
|
-
touch $HOME/.gem/credentials
|
|
91
|
-
chmod 0600 $HOME/.gem/credentials
|
|
92
|
-
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
93
|
-
gem build *.gemspec
|
|
94
|
-
gem push *.gem
|
|
95
|
-
env:
|
|
96
|
-
GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
|
data/test/proto/addressbook
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
people: [{name:"Masahiro", id: 1337, email: "repeatedly _at_ gmail.com"}]
|
data/test/proto/addressbook.bin
DELETED