fluent-plugin-formatter-protobuf 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5522a7376f600ab04c25874fed66cf1e3fa80488aff7917a0b4a20f2ae23e5a7
-  data.tar.gz: 56ff168034323c60b1c6482aa9be5d33540d6e1cdf58e8cb22f92cea5bfd8a6c
+  metadata.gz: 3e212013e4ca4c6d50c19b1d2296e06626750039cb0b22f80ea255be37cd4d9f
+  data.tar.gz: 30a007dba4d306d022d45cd5cc7230b7c3b9ba3edbb512b9e3924407588af03c
 SHA512:
-  metadata.gz: ef2e9c5998ed03fb9d24e5107185107cc37d9b5a803b1ab7fb2cedd624d55db04d62f32a55b4c80fb3422617f427ad48d168788b43891e6883ac908bbc9d90f5
-  data.tar.gz: 46e42467e1df6ec0c4c9fb45312141882da3a23c181cf408be0eb314edea2eff6708d89629210b41ff3f5d979070f083dd2018cddf45bd2e332d2b8b8d3eb428
+  metadata.gz: e3c804a6a7cf38e50fb273e17437c7b14a57351f7e63483a24873b82c78adc7307de33f17c1f33504ac361a9b5b32488aabfcabb157b26c5214a9851f054d234
+  data.tar.gz: 21adce1af4c987ddb7b3fbde52e3599cb98a925656550627ae625a8c962c88d67122db3ca55ba0c22a8875e86e7d8b9d6f49dc1d87acf311f6f380eec9c3aad0

data/.github/workflows/build-and-test.yml

@@ -0,0 +1,54 @@
+name: Build and test
+
+on:
+  push:
+
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake build"

data/.github/workflows/publish.yml

@@ -0,0 +1,107 @@
+name: Publish
+
+on:
+  push:
+    tags:
+      - v0.*
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake lint:check"
+
+  test:
+    name: Unit test
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: "Unit test"
+        run: "bundle exec rake test:unit"
+
+  publish-to-gpr:
+    needs:
+      - lint
+      - test
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to GPR
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+        env:
+          GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+          OWNER: ${{ github.repository_owner }}
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+
+  publish-to-ruby-gems:
+    needs:
+      - lint
+      - test
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+
+      - name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem
+        env:
+          GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.gitignore

@@ -1,2 +1 @@
-yarn-error.log
-node_modules
+*.gem

data/.rubocop.yml

@@ -1,13 +1,16 @@
+require: rubocop-rake
 AllCops:
   NewCops: enable
 Style/StringLiterals:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/HashSyntax:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Style/FrozenStringLiteralComment:
   Exclude:
-    - 'test/proto/addressbook_pb.rb'
+    - '**/*_pb.rb'
 Metrics/MethodLength:
-  Max: 100
+  Max: 100
+  Exclude:
+    - '**/*_pb.rb'

data/Gemfile.lock

@@ -1,13 +1,14 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-formatter-protobuf (0.0.3)
+    fluent-plugin-formatter-protobuf (0.0.4)
       fluentd (>= 1.0, < 2)
       google-protobuf (~> 3.18)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    ast (2.4.2)
     concurrent-ruby (1.1.9)
     cool.io (1.7.1)
     fluentd (1.14.1)
@@ -25,8 +26,28 @@ GEM
     google-protobuf (3.18.1-x86_64-linux)
     http_parser.rb (0.7.0)
     msgpack (1.4.2)
+    parallel (1.21.0)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
     power_assert (2.0.1)
+    rainbow (3.0.0)
     rake (13.0.6)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rubocop (1.22.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.12.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.12.0)
+      parser (>= 3.0.1.1)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    ruby-progressbar (1.11.0)
     serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -37,6 +58,7 @@ GEM
       concurrent-ruby (~> 1.0)
     tzinfo-data (1.2021.3)
       tzinfo (>= 1.0.0)
+    unicode-display_width (2.1.0)
     webrick (1.7.0)
     yajl-ruby (1.4.1)
 
@@ -47,6 +69,8 @@ DEPENDENCIES
   bundler (~> 2.2.22)
   fluent-plugin-formatter-protobuf!
   rake (~> 13.0.3)
+  rubocop (~> 1.22.1)
+  rubocop-rake (~> 0.6.0)
   test-unit (~> 3.3.7)
 
 BUNDLED WITH

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-formatter-protobuf
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf.svg)](https://badge.fury.io/rb/fluent-plugin-formatter-protobuf)
+
 [Fluentd](https://fluentd.org/) formatter plugin to format messages into Protobuf v3 encoded binary.
 
 ## Installation
@@ -24,6 +26,43 @@ And then execute:
 $ bundle
 ```
 
+## Not so quick start
+
+1. Generate the protobuf ruby methods
+   1. e.g `protoc --proto_path=. --ruby_out=. ./log.proto`
+2. Modify the `<format>` section to include `message_name`, which is your Protobuf message name, and `include_paths`, the path where the generated Ruby types are stored
+
+
+## Example
+
+```fluentd
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type file
+
+    path /opt/fluent-plugin-formatter-protobuf/out
+
+    <buffer>
+        @type memory
+    </buffer>
+
+    <format>
+        @type protobuf
+        message_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>
+```
+
 ## Configuration
 
 You can generate configuration template:

data/RELEASE.md

@@ -0,0 +1,9 @@
+# Release
+
+## Instructions
+To release the gem to Github package manager and Ruby Gems,
+
+1. Update `lib/fluent/version.rb`
+2. Update `Gemfile.lock`
+3. Create a pull request and merge it into `main`
+4. Manually creating a release on Github with a tag indicating the new version number e.g. `vMAJOR.MINOR.PATCH`

data/Rakefile

@@ -44,6 +44,11 @@ namespace :lint do
   end
 end
 
+desc 'Building ruby gem'
+task :build do
+  sh 'gem build *.gemspec'
+end
+
 desc 'Listing all tasks'
 task :help do
   sh 'rake --tasks'

data/fluent-plugin-formatter-protobuf.gemspec

@@ -6,7 +6,7 @@ require 'fluent/plugin/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-formatter-protobuf'
-  spec.version = Fluent::Plugin::ProtobufFormatter::VERSION
+  spec.version = Fluent::Plugin::VERSION
   spec.authors = ['Ray Tung']
   spec.email = ['code@raytung.net']
 
@@ -30,6 +30,8 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 2.2.22'
   spec.add_development_dependency 'rake', '~> 13.0.3'
+  spec.add_development_dependency 'rubocop', '~> 1.22.1'
+  spec.add_development_dependency 'rubocop-rake', '~> 0.6.0'
   spec.add_development_dependency 'test-unit', '~> 3.3.7'
   spec.add_runtime_dependency 'fluentd', ['>= 1.0', '< 2']
   spec.add_runtime_dependency 'google-protobuf', ['~> 3.18']

data/lib/fluent/plugin/version.rb

@@ -2,8 +2,6 @@
 
 module Fluent
   module Plugin
-    module ProtobufFormatter
-      VERSION = '0.0.3'
-    end
+    VERSION = '0.0.4'
   end
 end

data/test/integration/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'fluentd', '1.14.1'
+gem 'fluent-plugin-formatter-protobuf', '0.0.3'

data/test/integration/Gemfile.lock

@@ -0,0 +1,45 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    fluent-plugin-formatter-protobuf (0.0.3)
+      fluentd (>= 1.0, < 2)
+      google-protobuf (~> 3.18)
+    fluentd (1.14.1)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    google-protobuf (3.18.1)
+    google-protobuf (3.18.1-x86_64-linux)
+    http_parser.rb (0.7.0)
+    msgpack (1.4.2)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.3)
+      tzinfo (>= 1.0.0)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  fluent-plugin-formatter-protobuf (= 0.0.3)
+  fluentd (= 1.14.1)
+
+BUNDLED WITH
+   2.2.29

data/test/integration/README.md

@@ -0,0 +1,12 @@
+# Integration test
+
+A _manual_ integration test.
+
+### Instructions
+
+1. Open a new terminal and run`$ ./run.sh`
+2. Open a separate terminal and run `echo '{"timestamp": 1634386250, "message":"hello!"}\n' | netcat 127.0.0.1 5170`
+3. Somehow flush data (I simply stop the docker container)
+4. Verify out data (It should look like `out.YYYYMMDD_N.log`) with `protoc`
+   1. `protoc --decode_raw < out.YYYYMMDD_N.log`
+   2. `cat out.YYYYMMDD_N.log | protoc --decode=Log --proto_path=. log.proto`

data/test/integration/fluent.conf

@@ -0,0 +1,25 @@
+<source>
+    @type tcp
+    tag tcp.events
+    port 5170
+    delimiter "\n"
+    <parse>
+        @type json
+    </parse>
+</source>
+
+<match tcp.events>
+    @type file
+
+    path /opt/fluent-plugin-formatter-protobuf/out
+
+    <buffer>
+        @type memory
+    </buffer>
+
+    <format>
+        @type protobuf
+        message_name "Log"
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/log_pb.rb"]
+    </format>
+</match>

data/test/integration/log.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message Log {
+  int64 timestamp = 1;
+  string message = 2;
+}

data/test/integration/log_pb.rb

@@ -0,0 +1,15 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: log.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("log.proto", :syntax => :proto3) do
+    add_message "Log" do
+      optional :timestamp, :int64, 1
+      optional :message, :string, 2
+    end
+  end
+end
+
+Log = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Log").msgclass

data/test/integration/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+docker run -it --rm \
+  -v "${PWD}:/opt/fluent-plugin-formatter-protobuf/" \
+  -w '/opt/fluent-plugin-formatter-protobuf/' \
+  -e OUT_FILE="./out" \
+  -p 5170:5170 \
+  ruby:2.7.1 sh -c 'bundle install && fluentd -c /opt/fluent-plugin-formatter-protobuf/fluent.conf'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-formatter-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Ray Tung
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 13.0.3
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.22.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.22.1
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -94,7 +122,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".github/workflows/gem-push.yml"
+- ".github/workflows/build-and-test.yml"
+- ".github/workflows/publish.yml"
 - ".gitignore"
 - ".rubocop.yml"
 - ".ruby-version"
@@ -102,21 +131,27 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- RELEASE.md
 - Rakefile
 - docker-compose.yaml
 - docker-run.sh
 - fluent-plugin-formatter-protobuf.gemspec
 - lib/fluent/plugin/formatter_protobuf.rb
 - lib/fluent/plugin/version.rb
-- package.json
 - test/helper.rb
+- test/integration/Gemfile
+- test/integration/Gemfile.lock
+- test/integration/README.md
+- test/integration/fluent.conf
+- test/integration/log.proto
+- test/integration/log_pb.rb
+- test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
 - test/proto/addressbook
 - test/proto/addressbook.bin
 - test/proto/addressbook.proto
 - test/proto/addressbook_pb.rb
-- yarn.lock
 homepage: https://github.com/raytung/fluent-plugin-formatter-protobuf
 licenses:
 - Apache-2.0
@@ -145,6 +180,13 @@ specification_version: 4
 summary: Protobuf formatter for Fluentd
 test_files:
 - test/helper.rb
+- test/integration/Gemfile
+- test/integration/Gemfile.lock
+- test/integration/README.md
+- test/integration/fluent.conf
+- test/integration/log.proto
+- test/integration/log_pb.rb
+- test/integration/run.sh
 - test/plugin/test_formatter_protobuf.rb
 - test/proto/README.md
 - test/proto/addressbook

data/.github/workflows/gem-push.yml

@@ -1,19 +0,0 @@
-name: Publish Package
-on:
-  push:
-    branches:
-      - "main"
-  pull_request:
-    branches:
-      - "main"
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.3.5
-      - uses: actions/setup-node@v2.4.1
-        with:
-          node-version: "16"
-      - uses: ruby/setup-ruby@v1.83.0
-        with:
-          ruby-version: "2.7.1"

data/package.json

@@ -1,58 +0,0 @@
-{
-  "name": "fluent-plugin-formatter-protobuf",
-  "version": "0.0.3",
-  "description": "Protobuf formatter for Fluentd",
-  "main": "dist/index.js",
-  "repository": "git@github.com:raytung/fluent-plugin-formatter-protobuf.git",
-  "author": "Ray Tung <code@raytung.net>",
-  "license": "Apache-2.0",
-  "private": false,
-  "scripts": {
-    "release": "semantic-release"
-  },
-  "devDependencies": {
-    "@semantic-release/changelog": "^6.0.0",
-    "@semantic-release/commit-analyzer": "^9.0.1",
-    "@semantic-release/git": "^10.0.0",
-    "@semantic-release/github": "^8.0.1",
-    "@semantic-release/npm": "^8.0.0",
-    "@semantic-release/release-notes-generator": "^10.0.2",
-    "semantic-release": "^18.0.0",
-    "semantic-release-rubygem": "^1.2.0"
-  },
-  "release": {
-    "branches": [
-      "main"
-    ],
-    "plugins": [
-      "@semantic-release/commit-analyzer",
-      "@semantic-release/release-notes-generator",
-      [
-        "@semantic-release/npm",
-        {
-          "npmPublish": false
-        }
-      ],
-      [
-        "semantic-release-rubygem",
-        {
-          "updateGemfileLock": true
-        }
-      ],
-      "@semantic-release/changelog",
-      [
-        "@semantic-release/git",
-        {
-          "assets": [
-            "package.json",
-            "yarn.lock",
-            "lib/fluent/plugin/version.rb",
-            "CHANGELOG.md"
-          ],
-          "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
-        }
-      ],
-      "@semantic-release/github"
-    ]
-  }
-}