fluent-plugin-formatter-protobuf 0.1.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca1a8fea393af2eed56eac08ac8fe4f8122d0c1e5b5d36099580ff9e0c962c67
-  data.tar.gz: 9670c004adcabfe98e6ae98fe46a45ebe2a55e08df52d08e169fa149af2d032b
+  metadata.gz: 1be52a5c380fb27bcda9d3773404892a45b787d81014934071dc272ad4b07249
+  data.tar.gz: ae15b5a83abe6bd2d6af1ebd41b4f47711f5f6ebda413d2b5ac799f900a140e5
 SHA512:
-  metadata.gz: 8e8f996d0940c53666eae2cea2b7d9b47a6d57a15d6c2038c7d83accf592d89018e8c864731e5b574449cc7c5d2a9b4ea6f4153cbb9dbdd4de84cc3e5c8a57ec
-  data.tar.gz: 6b398f92c55ad9f8f6bc92b8da8947a887fa8bc3b3bd838c7973552e8aa12e353e04fcd76e13f9d314ace7429ced3c47b877b69f4d8b6270f7aefcfc0cb72758
+  metadata.gz: d16537f2c02989d35d00edfed6db6c0fd4c4c5258ca59767a5682ab819863f25e7663a87c473fa490679bc3806de37237df0ec52e9bf918bdc05768ea0a0707f
+  data.tar.gz: 9f2d7044287ce06a11af7c91396fb1436fa695faf4537290488501fef631189662f19f27f636cc3e9f505734422a9f313aac2faf1431866aa863116322023f47

data/.github/workflows/release-please.yml

@@ -0,0 +1,48 @@
+on:
+  push:
+    branches:
+      - main
+name: release-please
+jobs:
+  release-please:
+    environment: main
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/release-please-action@v2
+        id: release
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: ruby
+          version-file: "lib/fluent/plugin/version.rb"
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to GPR
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+        if: ${{ steps.release.outputs.release_created }}
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        if: ${{ steps.release.outputs.release_created }}

data/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "0.2.0"
+}

data/.rubocop.yml

@@ -13,4 +13,8 @@ Style/FrozenStringLiteralComment:
 Metrics/MethodLength:
   Max: 100
   Exclude:
-    - '**/*_pb.rb'
+    - '**/*_pb.rb'
+Metrics/ClassLength:
+  Max: 140
+Layout/LineLength:
+  Max: 150

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## [0.4.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.3.0...v0.4.0) (2022-04-11)
+
+
+### Features
+
+* Add configuration `ignore_unknown_fields` when decoding JSON ([734c638](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/commit/734c638cd2e2405c52def5babbd61fa4d69b8c08))
+
+## [0.3.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.2.0...v0.3.0) (2022-01-13)
+
+
+### Features
+
+* Add new configuration - `format_field` ([797789b](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/commit/797789b38d5a599c2c107979beb7edf66e384bd5))

data/README.md

@@ -88,11 +88,14 @@ This plugin only supports Protobuf v3.
 
 ## Configuration
 
-|parameter| type               | description                                                   | default |
-|---|--------------------|---------------------------------------------------------------|---------|
-|include_paths| array (optional)   | Generated Ruby Protobuf class files path                      | `[]`    |
-|class_name| string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary ||
-|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                              | `false`  |
+| parameter             | type               | description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | default   |
+|-----------------------|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
+| class_name            | string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary                                                                                                                                                                                                                                                                                                                                                                                                                                                        ||
+| decode_json           | boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                                                                                                                                                                                                                                                                                                                                                   | `false`   |
+| ignore_unknown_fields | boolean (optional) | Ignore unknown fields when decoding JSON. This parameter is only used if `decode_json` is `true`                                                                                                                                                                                                                                                                                                                                                                                                                     | `true`    |
+| format_field          | string (optional)  | When defined, the plugin will only serialise the record in the given field rather than the whole record. This is potentially useful if you intend to use this formatter with the Kafka output plugin (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not wish to include those data in the resulting proto3 binary. Defaults to serialising the entire record. | `''`      |
+| include_paths         | array (required)   | Generated Ruby Protobuf class files path                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | `[]`      |
+| require_method        | string (optional)  | Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files are provided by a Ruby Gem, you may want to use 'require'. If you are providing the generated files as files, you may want to use 'require_relative'                                                                                                                                                                                                                                          | `require` |
 
 ## Tips
 

data/RELEASE.md

@@ -1,9 +1,4 @@
 # Release
 
-## Instructions
-To release the gem to Github package manager and Ruby Gems,
-
-1. Update `lib/fluent/plugin/version.rb`
-2. Create a pull request and merge it into `main`
-3. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`
-4. GitHub Actions will automatically run and publish to RubyGems and GitHub repository
+This repository follow conventional commit and semantic versioning, and the release to RubyGems.org and GitHub Ruby
+ Repository is managed by [release-please](https://github.com/google-github-actions/release-please-action).

data/lib/fluent/plugin/formatter_protobuf.rb

@@ -30,12 +30,50 @@ module Fluent
     class ProtobufFormatter < Fluent::Plugin::Formatter
       Fluent::Plugin.register_formatter('protobuf', self)
 
-      config_param :include_paths, :array, default: [], desc: 'Generated Ruby Protobuf class files path'
-
-      config_param :class_name, :string, desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
-
-      config_param :decode_json, :bool, default: false,
-                                        desc: 'Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary' # rubocop:disable Layout/LineLength
+      config_param :class_name,
+                   :string,
+                   desc: 'Ruby Protobuf class name. Used to encode into Protobuf binary'
+
+      config_param :decode_json,
+                   :bool,
+                   default: false,
+                   desc: <<~DESC
+                     Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary'
+                   DESC
+
+      config_param :ignore_unknown_fields,
+                   :bool,
+                   default: true,
+                   desc: <<~DESC
+                     Ignore unknown fields when decoding JSON. This parameter is only used if `decode_json` is `true`
+                   DESC
+
+      config_param :format_field,
+                   :string,
+                   default: '',
+                   desc: <<~DESC
+                     When defined, the plugin will only serialise the record in the given field rather than the whole record.
+                      This is potentially useful if you intend to use this formatter with the Kafka output plugin
+                      (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains
+                      a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not
+                      wish to include those data in the resulting proto3 binary.
+
+                      Defaults to serialising the whole record.
+                   DESC
+
+      config_param :include_paths,
+                   :array,
+                   default: [],
+                   desc: 'Generated Ruby Protobuf class files path'
+
+      config_param :require_method,
+                   :string,
+                   default: 'require',
+                   desc: <<~DESC
+                     Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files
+                     are provided by a Ruby Gem, you would want to use \'require\'. If you are providing the generated files as files, you
+                     may want to use \'require_relative\''
+                   DESC
 
       def configure(conf)
         super(conf)
@@ -55,18 +93,21 @@ module Fluent
       end
 
       def format(_tag, _time, record)
-        protobuf_msg = @decode_json ? @protobuf_class.decode_json(Oj.dump(record)) : @protobuf_class.new(record)
+        format_record = @format_field == '' ? record : record[@format_field]
+
+        protobuf_msg = if @decode_json
+                         @protobuf_class.decode_json(Oj.dump(format_record),
+                                                     { ignore_unknown_fields: @ignore_unknown_fields })
+                       else
+                         @protobuf_class.new(format_record)
+                       end
         @protobuf_class.encode(protobuf_msg)
       end
 
       def require_proto!(filename)
-        if Pathname.new(filename).absolute?
-          require filename
-        else
-          require_relative filename
-        end
+        Kernel.method(@require_method.to_sym).call(filename)
       rescue LoadError => e
-        raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.inspect}"
+        raise Fluent::ConfigError, "Unable to load file '#{filename}'. Reason: #{e.message}"
       end
     end
   end

data/lib/fluent/plugin/version.rb

@@ -2,6 +2,6 @@
 
 module Fluent
   module Plugin
-    VERSION = '0.1.1'
+    VERSION = '0.4.0'
   end
 end

data/release-please-config.json

@@ -0,0 +1,13 @@
+{
+  "packages": {
+    ".": {
+      "releaseType": "ruby",
+      "draft": false,
+      "prerelease": false,
+      "bumpMinorPreMajor": true,
+      "bumpPatchForMinorPreMajor": true,
+      "changelogPath": "CHANGELOG.md",
+      "versioning": "default"
+    }
+  }
+}

data/test/integration/Gemfile.lock

@@ -18,8 +18,8 @@ GEM
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    google-protobuf (3.19.1)
-    google-protobuf (3.19.1-x86_64-linux)
+    google-protobuf (3.19.2)
+    google-protobuf (3.19.2-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
     serverengine (2.2.4)

data/test/plugin/test_formatter_protobuf.rb

@@ -13,6 +13,7 @@ class ProtobufFormatterTest < Test::Unit::TestCase
   # Relative to the plugin file
   VALID_INCLUDE_PATHS_RELATIVE = '../../../test/proto/addressbook_pb.rb'
 
+  # rubocop:disable Metrics/BlockLength
   sub_test_case 'configure' do
     test 'fail if include_paths is empty' do
       assert_raise(Fluent::ConfigError) do
@@ -34,7 +35,11 @@ class ProtobufFormatterTest < Test::Unit::TestCase
 
     test 'success if given valid relative paths in include paths' do
       assert_nothing_raised do
-        create_driver({ class_name: 'tutorial.AddressBook', include_paths: [VALID_INCLUDE_PATHS_RELATIVE] })
+        create_driver({
+                        class_name: 'tutorial.AddressBook',
+                        include_paths: [VALID_INCLUDE_PATHS_RELATIVE],
+                        require_method: 'require_relative'
+                      })
       end
     end
 
@@ -44,38 +49,97 @@ class ProtobufFormatterTest < Test::Unit::TestCase
       end
     end
   end
-
-  stub_ruby_hash = { people: [{ name: 'Masahiro', id: 1337,
-                                email: 'repeatedly _at_ gmail.com',
-                                last_updated: {
-                                  seconds: 1_638_489_505,
-                                  nanos: 318_000_000
-                                } }] }
+  # rubocop:enable Metrics/BlockLength
+
+  stub_ruby_hash = { 'people' => [{ 'name' => 'Masahiro', 'id' => 1337,
+                                    'email' => 'repeatedly _at_ gmail.com',
+                                    'last_updated' => {
+                                      'seconds' => 1_638_489_505,
+                                      'nanos' => 318_000_000
+                                    } }] }
+  # rubocop:disable Metrics/BlockLength
   sub_test_case 'format' do
     test 'encodes into Protobuf binary' do
       formatter = create_formatter({ class_name: 'tutorial.AddressBook',
                                      include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
 
+      formatted = formatter.format('some-tag', 1234, stub_ruby_hash)
+      address_book = Tutorial::AddressBook.new(stub_ruby_hash)
+      assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
+    end
+
+    test 'encodes a particular field instead of the entire record if format_field is defined' do
+      formatter = create_formatter({ class_name: 'tutorial.AddressBook',
+                                     include_paths: VALID_INCLUDE_PATHS_ABSOLUTE,
+                                     format_field: 'data' })
+
       formatted = formatter.format('some-tag', 1234,
-                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
-                                                last_updated: { seconds: 1_638_489_505, nanos: 318_000_000 } }] })
+                                   {
+                                     'topic' => 'some-kafka-topic',
+                                     'headers' => {},
+                                     'data' => stub_ruby_hash
+                                   })
       address_book = Tutorial::AddressBook.new(stub_ruby_hash)
       assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
     end
 
-    test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true' do
+    test 'encodes Protobuf JSON format into Protobuf binary if config_param decode_json is true and if incoming JSON contains unknown fields' do
       formatter = create_formatter({ class_name: 'tutorial.AddressBook',
                                      decode_json: true,
                                      include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
 
       formatted = formatter.format('some-tag', 1234,
-                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
-                                                last_updated: '2021-12-02T23:58:25.318Z' }] })
+                                   {
+                                     'people' => [
+                                       {
+                                         'name' => 'Masahiro',
+                                         'id' => 1337,
+                                         'email' => 'repeatedly _at_ gmail.com',
+                                         'last_updated' => '2021-12-02T23:58:25.318Z',
+                                         'some-unknown-fields' => 'this field is not specified in the .proto message'
+                                       }
+                                     ]
+                                   })
 
       address_book = Tutorial::AddressBook.new(stub_ruby_hash)
       assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
     end
+
+    test 'throws exception when formatting JSON with unknown fields and ignore_unknown_fields is `false`' do
+      formatter = create_formatter({ class_name: 'tutorial.AddressBook',
+                                     decode_json: true,
+                                     ignore_unknown_fields: false,
+                                     include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
+
+      assert_raise(Google::Protobuf::ParseError) do
+        formatter.format('some-tag', 1234,
+                         {
+                           'people' => [
+                             {
+                               'name' => 'Masahiro',
+                               'id' => 1337,
+                               'email' => 'repeatedly _at_ gmail.com',
+                               'last_updated' => '2021-12-02T23:58:25.318Z',
+                               'some-unknown-fields' => 'this field is not specified in the .proto message'
+                             }
+                           ]
+                         })
+      end
+    end
+
+    test 'encodes Ruby hash into Protobuf binary if generated files are provided by a Gem' do
+      formatter = create_formatter({
+                                     class_name: 'google.protobuf.Duration',
+                                     # Provided by the google-protobuf gem
+                                     include_paths: ['google/protobuf/duration_pb'],
+                                     require_method: 'require'
+                                   })
+      formatted = formatter.format('some-tag', 1234, { seconds: 1, nanos: 340_012 })
+      duration = Google::Protobuf::Duration.new({ seconds: 1, nanos: 340_012 })
+      assert_equal(Google::Protobuf::Duration.encode(duration), formatted)
+    end
   end
+  # rubocop:enable Metrics/BlockLength
 
   private
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-formatter-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Ray Tung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-06 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -138,10 +138,12 @@ extra_rdoc_files: []
 files:
 - ".github/workflows/build-and-test.yml"
 - ".github/workflows/codeql-analysis.yml"
-- ".github/workflows/publish.yml"
+- ".github/workflows/release-please.yml"
 - ".gitignore"
+- ".release-please-manifest.json"
 - ".rubocop.yml"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -152,6 +154,7 @@ files:
 - fluent-plugin-formatter-protobuf.gemspec
 - lib/fluent/plugin/formatter_protobuf.rb
 - lib/fluent/plugin/version.rb
+- release-please-config.json
 - test/helper.rb
 - test/integration/Gemfile
 - test/integration/Gemfile.lock

data/.github/workflows/publish.yml

@@ -1,96 +0,0 @@
-name: Publish
-
-on:
-  push:
-    tags:
-      - v0.*
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-      - name: "Unit test"
-        run: "bundle exec rake lint:check"
-
-  test:
-    name: Unit test
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-      - name: "Unit test"
-        run: "bundle exec rake test:unit"
-
-  publish-to-gpr:
-    needs:
-      - lint
-      - test
-    name: Build + Publish to GPR
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Publish to GPR
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem build *.gemspec
-          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
-        env:
-          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
-          OWNER: ${{ github.repository_owner }}
-
-  publish-to-ruby-gems:
-    needs:
-      - lint
-      - test
-    name: Build + Publish to RubyGems
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Publish to RubyGems
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem build *.gemspec
-          gem push *.gem
-        env:
-          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"