fluent-plugin-formatter-protobuf 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab1edaa0c55c3e7c3174c18c682aa96a6a07056d6017dc5180cea1c988a03189
-  data.tar.gz: 8a10cffebdd8e605465f61ba8e30dbcf6da7b332ecb44f8bd08b60cf2dbf4539
+  metadata.gz: e6218edd36cd2e7186b2def09700237885d5851931a15ad3859b8eecdc8fd1ca
+  data.tar.gz: f609d96e5a3dd326ee0cc5d9a9638942ed9f9f063f34bca7ad696176ed58f589
 SHA512:
-  metadata.gz: 6d0a30cb1342b7fdcbe7057d223dacd150dba565d11c29ff2b4395fb35347e76f21463d23af5bbbe6b27882cbbd700ee4b1e5a5eb82d153093c37be812428752
-  data.tar.gz: 9865e62b40999dcbfcb82d30ab2363f8f704d2f0b8f07612f42165b633da83b3f2adc6df1a948321d75e182fa16abbb0b610da9dbba81a9f5a089376488e0ac9
+  metadata.gz: de97c9eed15e2ff1f86a4042412ad59efedad063a6957f052828b5efd5a3e6c888e9a1aa3f2f0c4cc1b1c33b3906a48d91dbd4d1b8dd855ece6c24db2be2742d
+  data.tar.gz: 45365b906912baf789221d565b039a44a4c97ac596518da587778ece2e92856b027a6ac5768cfae8fd64c9dd5b4032195ba2fc874c73cb1b804b9c93286c7682

data/.github/workflows/release-please.yml

@@ -0,0 +1,48 @@
+on:
+  push:
+    branches:
+      - main
+name: release-please
+jobs:
+  release-please:
+    environment: main
+    runs-on: ubuntu-latest
+    steps:
+      - uses: google-github-actions/release-please-action@v2
+        id: release
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          release-type: ruby
+          version-file: "lib/fluent/plugin/version.rb"
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Ruby 2.5
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to GPR
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+        if: ${{ steps.release.outputs.release_created }}
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        if: ${{ steps.release.outputs.release_created }}

data/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+  ".": "0.2.0"
+}

data/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## [0.3.0](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/compare/v0.2.0...v0.3.0) (2022-01-13)
+
+
+### Features
+
+* Add new configuration - `format_field` ([797789b](https://www.github.com/fluent-plugins-nursery/fluent-plugin-formatter-protobuf/commit/797789b38d5a599c2c107979beb7edf66e384bd5))

data/README.md

@@ -88,12 +88,13 @@ This plugin only supports Protobuf v3.
 
 ## Configuration
 
-|parameter| type               | description                                                                                                                                                                                                                                                                 | default   |
-|---|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
-|class_name| string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary                                                                                                                                                                                                               ||
-|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                                                                                                          | `false`   |
-|include_paths| array (required)   | Generated Ruby Protobuf class files path                                                                                                                                                                                                                                    | `[]`      |
-|require_method| string (optional)  | Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files are provided by a Ruby Gem, you may want to use 'require'. If you are providing the generated files as files, you may want to use 'require_relative' | `require` |
+|parameter| type               | description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | default |
+|---|--------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|
+|class_name| string (required)  | Ruby Protobuf class name. Used to encode into Protobuf binary                                                                                                                                                                                                                                                                                                                                                                                                                                                        ||
+|decode_json| boolean (optional) | Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary                                                                                                                                                                                                                                                                                                                                                                                   | `false` |
+|format_field| string (optional)  | When defined, the plugin will only serialise the record in the given field rather than the whole record. This is potentially useful if you intend to use this formatter with the Kafka output plugin (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not wish to include those data in the resulting proto3 binary. Defaults to serialising the entire record. | `''`      |
+|include_paths| array (required)   | Generated Ruby Protobuf class files path                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | `[]`    |
+|require_method| string (optional)  | Determine how to bring the generated Ruby Protobuf class files into scope. If your generated Ruby Protobuf class files are provided by a Ruby Gem, you may want to use 'require'. If you are providing the generated files as files, you may want to use 'require_relative'                                                                                                                                                                                                                                          | `require` |
 
 ## Tips
 

data/RELEASE.md

@@ -1,9 +1,4 @@
 # Release
 
-## Instructions
-To release the gem to Github package manager and Ruby Gems,
-
-1. Update `lib/fluent/plugin/version.rb`
-2. Create a pull request and merge it into `main`
-3. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`
-4. GitHub Actions will automatically run and publish to RubyGems and GitHub repository
+This repository follow conventional commit and semantic versioning, and the release to RubyGems.org and GitHub Ruby
+ Repository is managed by [release-please](https://github.com/google-github-actions/release-please-action).

data/lib/fluent/plugin/formatter_protobuf.rb

@@ -41,6 +41,19 @@ module Fluent
                      Serializes record from canonical proto3 JSON mapping (https://developers.google.com/protocol-buffers/docs/proto3#json) into binary'
                    DESC
 
+      config_param :format_field,
+                   :string,
+                   default: '',
+                   desc: <<~DESC
+                     When defined, the plugin will only serialise the record in the given field rather than the whole record.
+                      This is potentially useful if you intend to use this formatter with the Kafka output plugin
+                      (https://github.com/fluent/fluent-plugin-kafka#output-plugin) for example, where your record contains
+                      a field to determine which Kafka topic to write to, or the Kafka headers to include, but you do not
+                      wish to include those data in the resulting proto3 binary.
+
+                      Defaults to serialising the whole record.
+                   DESC
+
       config_param :include_paths,
                    :array,
                    default: [],
@@ -73,7 +86,13 @@ module Fluent
       end
 
       def format(_tag, _time, record)
-        protobuf_msg = @decode_json ? @protobuf_class.decode_json(Oj.dump(record)) : @protobuf_class.new(record)
+        format_record = @format_field == '' ? record : record[@format_field]
+
+        protobuf_msg = if @decode_json
+                         @protobuf_class.decode_json(Oj.dump(format_record))
+                       else
+                         @protobuf_class.new(format_record)
+                       end
         @protobuf_class.encode(protobuf_msg)
       end
 

data/lib/fluent/plugin/version.rb

@@ -2,6 +2,6 @@
 
 module Fluent
   module Plugin
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
   end
 end

data/release-please-config.json

@@ -0,0 +1,13 @@
+{
+  "packages": {
+    ".": {
+      "releaseType": "ruby",
+      "draft": false,
+      "prerelease": false,
+      "bumpMinorPreMajor": true,
+      "bumpPatchForMinorPreMajor": true,
+      "changelogPath": "CHANGELOG.md",
+      "versioning": "default"
+    }
+  }
+}

data/test/integration/Gemfile.lock

@@ -18,8 +18,8 @@ GEM
       tzinfo-data (~> 1.0)
       webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    google-protobuf (3.19.1)
-    google-protobuf (3.19.1-x86_64-linux)
+    google-protobuf (3.19.2)
+    google-protobuf (3.19.2-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
     serverengine (2.2.4)

data/test/plugin/test_formatter_protobuf.rb

@@ -51,21 +51,34 @@ class ProtobufFormatterTest < Test::Unit::TestCase
   end
   # rubocop:enable Metrics/BlockLength
 
-  stub_ruby_hash = { people: [{ name: 'Masahiro', id: 1337,
-                                email: 'repeatedly _at_ gmail.com',
-                                last_updated: {
-                                  seconds: 1_638_489_505,
-                                  nanos: 318_000_000
-                                } }] }
+  stub_ruby_hash = { 'people' => [{ 'name' => 'Masahiro', 'id' => 1337,
+                                    'email' => 'repeatedly _at_ gmail.com',
+                                    'last_updated' => {
+                                      'seconds' => 1_638_489_505,
+                                      'nanos' => 318_000_000
+                                    } }] }
   # rubocop:disable Metrics/BlockLength
   sub_test_case 'format' do
     test 'encodes into Protobuf binary' do
       formatter = create_formatter({ class_name: 'tutorial.AddressBook',
                                      include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
 
+      formatted = formatter.format('some-tag', 1234, stub_ruby_hash)
+      address_book = Tutorial::AddressBook.new(stub_ruby_hash)
+      assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
+    end
+
+    test 'encodes a particular field instead of the entire record if format_field is defined' do
+      formatter = create_formatter({ class_name: 'tutorial.AddressBook',
+                                     include_paths: VALID_INCLUDE_PATHS_ABSOLUTE,
+                                     format_field: 'data' })
+
       formatted = formatter.format('some-tag', 1234,
-                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
-                                                last_updated: { seconds: 1_638_489_505, nanos: 318_000_000 } }] })
+                                   {
+                                     'topic' => 'some-kafka-topic',
+                                     'headers' => {},
+                                     'data' => stub_ruby_hash
+                                   })
       address_book = Tutorial::AddressBook.new(stub_ruby_hash)
       assert_equal(Tutorial::AddressBook.encode(address_book), formatted)
     end
@@ -76,8 +89,16 @@ class ProtobufFormatterTest < Test::Unit::TestCase
                                      include_paths: VALID_INCLUDE_PATHS_ABSOLUTE })
 
       formatted = formatter.format('some-tag', 1234,
-                                   { people: [{ name: 'Masahiro', id: 1337, email: 'repeatedly _at_ gmail.com',
-                                                last_updated: '2021-12-02T23:58:25.318Z' }] })
+                                   {
+                                     'people' => [
+                                       {
+                                         'name' => 'Masahiro',
+                                         'id' => 1337,
+                                         'email' => 'repeatedly _at_ gmail.com',
+                                         'last_updated' => '2021-12-02T23:58:25.318Z'
+                                       }
+                                     ]
+                                   })
 
       address_book = Tutorial::AddressBook.new(stub_ruby_hash)
       assert_equal(Tutorial::AddressBook.encode(address_book), formatted)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-formatter-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Ray Tung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-23 00:00:00.000000000 Z
+date: 2022-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -138,10 +138,12 @@ extra_rdoc_files: []
 files:
 - ".github/workflows/build-and-test.yml"
 - ".github/workflows/codeql-analysis.yml"
-- ".github/workflows/publish.yml"
+- ".github/workflows/release-please.yml"
 - ".gitignore"
+- ".release-please-manifest.json"
 - ".rubocop.yml"
 - ".ruby-version"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md
@@ -152,6 +154,7 @@ files:
 - fluent-plugin-formatter-protobuf.gemspec
 - lib/fluent/plugin/formatter_protobuf.rb
 - lib/fluent/plugin/version.rb
+- release-please-config.json
 - test/helper.rb
 - test/integration/Gemfile
 - test/integration/Gemfile.lock

data/.github/workflows/publish.yml

@@ -1,96 +0,0 @@
-name: Publish
-
-on:
-  push:
-    tags:
-      - v0.*
-
-jobs:
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-      - name: "Unit test"
-        run: "bundle exec rake lint:check"
-
-  test:
-    name: Unit test
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-      - name: "Unit test"
-        run: "bundle exec rake test:unit"
-
-  publish-to-gpr:
-    needs:
-      - lint
-      - test
-    name: Build + Publish to GPR
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Publish to GPR
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem build *.gemspec
-          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
-        env:
-          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
-          OWNER: ${{ github.repository_owner }}
-
-  publish-to-ruby-gems:
-    needs:
-      - lint
-      - test
-    name: Build + Publish to RubyGems
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Ruby 2.5
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Publish to RubyGems
-        run: |
-          mkdir -p $HOME/.gem
-          touch $HOME/.gem/credentials
-          chmod 0600 $HOME/.gem/credentials
-          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-          gem build *.gemspec
-          gem push *.gem
-        env:
-          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"