fluent-plugin-formatter-protobuf 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f49ca0a97713b1a0771cb9c73b38f49660b5fb4103a6d3f989140013fbd21bab
-  data.tar.gz: d547aaeef78fb13b4d5747dcf26fa27f1912e33ee6b4f699db401f335525fa94
+  metadata.gz: ca1a8fea393af2eed56eac08ac8fe4f8122d0c1e5b5d36099580ff9e0c962c67
+  data.tar.gz: 9670c004adcabfe98e6ae98fe46a45ebe2a55e08df52d08e169fa149af2d032b
 SHA512:
-  metadata.gz: c17daaf911ea6a58cdbd8bd39b25bdf500630a506ab29f384d1abcf01e178578bea74ecf4e9ebff6c0496bf67304324f49b00744d49911bd1db32177d7fbcaed
-  data.tar.gz: bc2167f0f05d20594b453a7a40ba25c8e72ae085ed6c72a14714e2cdf617e9050f650b1c1aa1b82b5626f2c83666ef4015c19149deed227c4e0d39d79df910f2
+  metadata.gz: 8e8f996d0940c53666eae2cea2b7d9b47a6d57a15d6c2038c7d83accf592d89018e8c864731e5b574449cc7c5d2a9b4ea6f4153cbb9dbdd4de84cc3e5c8a57ec
+  data.tar.gz: 6b398f92c55ad9f8f6bc92b8da8947a887fa8bc3b3bd838c7973552e8aa12e353e04fcd76e13f9d314ace7429ced3c47b877b69f4d8b6270f7aefcfc0cb72758

data/README.md

@@ -50,10 +50,13 @@ $ bundle
 2. Modify the `<format>` section to include `class_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
    1. Given protobuf class `Your::Protobuf::Class::Name` class should be given as `Your.Protobuf.Class.Name` in `class_name`. The exact name can be found in the generated Ruby files
 
+## Note
+
+This plugin only supports Protobuf v3.
 
 ## Example
 
-```fluentd
+```aconf
 <source>
     @type tcp
     tag tcp.events
@@ -65,12 +68,14 @@ $ bundle
 </source>
 
 <match tcp.events>
-    @type file
-
-    path /opt/fluent-plugin-formatter-protobuf/out
+    @type kafka2
+    
+    brokers "#{BROKER_ENDPOINTS}"
+    compression_codec lz4
 
-    <buffer>
+    <buffer topic>
         @type memory
+        flush_interval 5s
     </buffer>
 
     <format>
@@ -83,13 +88,53 @@ $ bundle
 
 ## Configuration
 
-|parameter|type|description|default|
-|---|---|---|---|
-|include_paths|array (optional)| Generated Ruby Protobuf class files path |`[]`|
-|class_name|string (required)| Ruby Protobuf class name. Used to encode into Protobuf binary||
+|parameter| type               | description                                                   | default |
+|---|--------------------|---------------------------------------------------------------|---------|
+|include_paths| array (optional)   | Generated Ruby Protobuf class files path                      | `[]`    |
+|class_name| string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary ||
+|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                              | `false`  |
+
+## Tips
+
+#### 1. I have multiple `_pb.rb` files and they depend on one another. When I use this plugin, I'm getting a `LoadError`.
+
+This happens because the plugin currently loads paths in `include_paths` sequentially. You can either sort the files in correct dependency order (which is cumbersome), or add to Ruby's `$LOAD_PATH`.
+
+For example, you have 2 generated ruby files, `a_pb.rb` and `b_pb.rb`, and both `a_pb.rb` depends on `b_pb.rb`.
+
+You can either order them in appropriate dependency order, such as
+
+```aconf
+<format>
+  @type protobuf
+  class_name "SomePackage.A"
+  include_paths ["/some/path/b_pb.rb", "/some/path/a_pb.rb"]
+</format>
+
+```
+
+or you can put the generated files in a directory say `/some/path/`, and add to Ruby's `$LOAD_PATH` via the `RUBYLIB` environment variable.
+
+e.g.
+
+```aconf
+<format>
+  @type protobuf
+  class_name "SomePackage.A"
+  include_paths ["/some/path/a_pb.rb"]
+</format>
+
+```
+
+```shell
+
+export RUBYLIB="${RUBYLIB}:/some/path/"
+fluentd \
+  --gemfile "/some/other/path/Gemfile" \
+  --config "/some/other/path/fluent.conf"
+
+```
 
-## Copyright
+## License
 
-* Copyright(c) 2021 - Ray Tung
-* License
-  * Apache License, Version 2.0
+[Apache License, Version 2.0](./LICENSE)

data/fluent-plugin-formatter-protobuf.gemspec

@@ -9,6 +9,7 @@ Gem::Specification.new do |spec|
   spec.version = Fluent::Plugin::VERSION
   spec.authors = ['Ray Tung']
   spec.email = ['code@raytung.net']
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
   spec.summary = 'Protobuf formatter for Fluentd'
   spec.description = 'This is a Fluentd formatter plugin designed to convert Protobuf JSON into Protobuf binary'
@@ -35,4 +36,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'test-unit', '~> 3.3'
   spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
   spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']
+  spec.add_runtime_dependency 'oj', '~> 3.13'
 end

data/lib/fluent/plugin/formatter_protobuf.rb

@@ -22,6 +22,7 @@ require 'google/protobuf'
 
 require 'fluent/env'
 require 'fluent/time'
+require 'oj'
 
 module Fluent
   module Plugin
@@ -33,6 +34,9 @@ module Fluent
 
       config_param :class_name, :string, desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
 
+      config_param :decode_json, :bool, default: false,
+                                        desc: 'Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary' # rubocop:disable Layout/LineLength
+
       def configure(conf)
         super(conf)
 
@@ -51,7 +55,7 @@ module Fluent
       end
 
       def format(_tag, _time, record)
-        protobuf_msg = @protobuf_class.new(record)
+        protobuf_msg = @decode_json ? @protobuf_class.decode_json(Oj.dump(record)) : @protobuf_class.new(record)
         @protobuf_class.encode(protobuf_msg)
       end
 

data/lib/fluent/plugin/version.rb

@@ -2,6 +2,6 @@
 
 module Fluent
   module Plugin
-    VERSION = '0.1.0'
+    VERSION = '0.1.1'
   end
 end

data/test/integration/Gemfile

@@ -2,5 +2,5 @@
 
 source 'https://rubygems.org'
 
-gem 'fluentd', '1.14.1'
-gem 'fluent-plugin-formatter-protobuf', '0.0.3'
+gem 'fluentd', '1.14.2'
+gem 'fluent-plugin-formatter-protobuf', '0.1.0'

data/test/integration/Gemfile.lock

@@ -3,23 +3,23 @@ GEM
   specs:
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
-    fluent-plugin-formatter-protobuf (0.0.3)
+    fluent-plugin-formatter-protobuf (0.1.0)
       fluentd (>= 1.0, < 2)
       google-protobuf (~> 3.18)
-    fluentd (1.14.1)
+    fluentd (1.14.2)
       bundler
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
       serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    google-protobuf (3.18.1)
-    google-protobuf (3.18.1-x86_64-linux)
+    google-protobuf (3.19.1)
+    google-protobuf (3.19.1-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
     serverengine (2.2.4)
@@ -28,7 +28,7 @@ GEM
     strptime (0.2.5)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    tzinfo-data (1.2021.3)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
     webrick (1.7.0)
     yajl-ruby (1.4.1)
@@ -38,8 +38,8 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  fluent-plugin-formatter-protobuf (= 0.0.3)
-  fluentd (= 1.14.1)
+  fluent-plugin-formatter-protobuf (= 0.1.0)
+  fluentd (= 1.14.2)
 
 BUNDLED WITH
    2.2.29

data/test/integration/fluent.conf

@@ -19,7 +19,7 @@
 
     <format>
         @type protobuf
-        message_name "Log"
+        class_name "Log"
         include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
     </format>
 </match>

data/test/plugin/test_formatter_protobuf.rb

@@ -45,15 +45,35 @@ class ProtobufFormatterTest < Test::Unit::TestCase
     end
   end
 
+  stub_ruby_hash = { people: [{ name: 'Masahiro', id: 1337,
+                                email: 'repeatedly _at_ gmail.com',
+                                last_updated: {
+                                  seconds: 1_638_489_505,
+                                  nanos: 318_000_000
+                                } }] }
   sub_test_case 'format' do
     test 'encodes into Protobuf binary' do
       formatter = create_formatter({ class_name: 'tutorial.AddressBook',
                                      include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
 
       formatted = formatter.format('some-tag', 1234,
-                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com' }] })
-      golden_file = File.binread(File.expand_path(File.join(__dir__, '..', 'proto', 'addressbook.bin')))
-      assert_equal(golden_file, formatted)
+                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
+                                                last_updated: { seconds: 1_638_489_505, nanos: 318_000_000 } }] })
+      address_book = Tutorial::AddressBook.new(stub_ruby_hash)
+      assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
+    end
+
+    test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true' do
+      formatter = create_formatter({ class_name: 'tutorial.AddressBook',
+                                     decode_json: true,
+                                     include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
+
+      formatted = formatter.format('some-tag', 1234,
+                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
+                                                last_updated: '2021-12-02T23:58:25.318Z' }] })
+
+      address_book = Tutorial::AddressBook.new(stub_ruby_hash)
+      assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-formatter-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Ray Tung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-30 00:00:00.000000000 Z
+date: 2021-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,6 +114,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.18'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
 description: This is a Fluentd formatter plugin designed to convert Protobuf JSON
   into Protobuf binary
 email:
@@ -148,14 +162,13 @@ files:
 - test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
-- test/proto/addressbook
-- test/proto/addressbook.bin
 - test/proto/addressbook.proto
 - test/proto/addressbook_pb.rb
 homepage: https://github.com/raytung/fluent-plugin-formatter-protobuf
 licenses:
 - Apache-2.0
 metadata:
+  rubygems_mfa_required: 'true'
   homepage_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
   source_code_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf
   changelog_uri: https://github.com/raytung/fluent-plugin-formatter-protobuf/blob/main/CHANGELOG.md
@@ -190,7 +203,5 @@ test_files:
 - test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
-- test/proto/addressbook
-- test/proto/addressbook.bin
 - test/proto/addressbook.proto
 - test/proto/addressbook_pb.rb

data/test/proto/addressbook

@@ -1 +0,0 @@
-people: [{name:"Masahiro", id: 1337, email: "repeatedly _at_ gmail.com"}]

data/test/proto/addressbook.bin

@@ -1,4 +0,0 @@
-
-(
-Masahiro�
-repeatedly _at_ gmail.com