fluent-plugin-protobuf-http 0.3.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/.github/dependabot.yml +6 -0
  3. data/.github/workflows/ci.yml +15 -26
  4. data/README.md +68 -39
  5. data/fluent-plugin-protobuf-http.gemspec +2 -1
  6. data/lib/fluent/plugin/in_protobuf_http.rb +4 -3
  7. metadata +10 -10
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c12d1da7ec8d1e801e5c9bc01ae3258e4c13ee6d361431721305e8a3778fe0d5
4
- data.tar.gz: 2571d4e0c3795fd37c87d07fcbfd80fc7179b7bab0c7d6760c07b3fc7a561e73
3
+ metadata.gz: a559c0013395560b44bd1a1e2ca12898e481b932b8860f1cf31f557544031458
4
+ data.tar.gz: 827f41577acb0bbf3f526047030a662a3069eaeafe5fa3a648e1027dfb41f2fc
5
5
  SHA512:
6
- metadata.gz: cfb5aba44bec8f2a9ef10514ef50f5fc2d14084ccd16a8b8bb8d25babc82f351b9bed58ee3c290e00af11fb6dbcb06f4c96d41c342ee67c33fd8b42f6db1399c
7
- data.tar.gz: e5e01488511a1509becf3f636e1ab0e653e243dbb44516279555963ed03b09d3333a92d9e8af07a9b9494f75b285adb97e8c7b3ac1f270111b50359054702f80
6
+ metadata.gz: 627665ee19fc340181a6c4e00912aa537b9b2f12bfff6c96c582180453302a6c88c756c583c9c7f003a19c57f9f48d799b912155dae8898e0a0bf54997030d9f
7
+ data.tar.gz: 6b85fda4bb44831b70023728cc8b10071d5a771f753f0c4054fa87a1f9ce650635dfd2a1fa9997c7e13ea798b65df8e874fd247de4d0339af20d7ebac6e93d02
data/.github/dependabot.yml ADDED
@@ -0,0 +1,6 @@
1
+ version: 2
2
+ updates:
3
+ - package-ecosystem: "github-actions"
4
+ directory: "/"
5
+ schedule:
6
+ interval: "weekly"
data/.github/workflows/ci.yml CHANGED
@@ -2,47 +2,36 @@ name: ci
2
2
 
3
3
  on:
4
4
  push:
5
- paths-ignore:
6
- - '**.md'
7
- - '.rubocop.yml'
5
+ branches: [main]
6
+ paths-ignore: ['**.md', '.rubocop.yml']
8
7
  pull_request:
9
- paths-ignore:
10
- - '**.md'
11
- - '.rubocop.yml'
8
+ branches: [main]
9
+ types: [opened, synchronize, reopened]
10
+ paths-ignore: ['**.md', '.rubocop.yml']
12
11
 
13
12
  jobs:
14
- run-tests:
15
- name: Run tests
13
+ build-and-test:
16
14
  strategy:
17
15
  matrix:
18
- os: [ubuntu-18.04]
19
- ruby-version: ['2.5', '2.6', '2.7']
16
+ os: [ubuntu-latest, macos-latest, windows-latest]
17
+ ruby-version: ['2.6', '2.7']
20
18
 
21
19
  runs-on: ${{ matrix.os }}
22
20
 
23
21
  steps:
24
- - uses: actions/checkout@v2
22
+ - name: Checkout [${{ github.repository }}]
23
+ uses: actions/checkout@v4
25
24
 
26
- - name: Set up Ruby
25
+ - name: Set up Ruby ${{ matrix.ruby-version }}
27
26
  uses: ruby/setup-ruby@v1
28
27
  with:
29
28
  ruby-version: ${{ matrix.ruby-version }}
30
29
  bundler-cache: true
31
30
 
32
- - name: Bundle install
33
- run: |
34
- gem install bundler && \
35
- bundle install --jobs 4 --retry 3
36
-
37
31
  - name: Install protoc
38
- run: |
39
- mkdir protoc && cd protoc
40
- wget https://github.com/protocolbuffers/protobuf/releases/download/v3.17.0/protoc-3.17.0-linux-x86_64.zip
41
- unzip protoc-3.17.0-linux-x86_64.zip
42
- chmod +x ./bin/protoc && sudo mv ./bin/protoc /usr/local/bin/protoc
43
- sudo mv ./include/* /usr/local/include/
44
- cd .. && rm -rf ./protoc
45
- protoc --version
32
+ uses: arduino/setup-protoc@master
33
+ with:
34
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
46
35
 
47
- - name: Tests
36
+ - name: Run tests
48
37
  run: bundle exec rake test
data/README.md CHANGED
@@ -1,24 +1,24 @@
1
1
  # fluent-plugin-protobuf-http
2
2
 
3
3
  [![ci](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/iamazeem/fluent-plugin-protobuf-http/actions/workflows/ci.yml)
4
- [![License: Apache](https://img.shields.io/badge/license-Apache-blue.svg?style=flat-square)](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
4
+ [![License: Apache](https://img.shields.io/badge/license-Apache-darkgreen.svg?style=flat-square)](https://github.com/iamAzeem/fluent-plugin-protobuf-http/blob/master/LICENSE)
5
5
  ![GitHub release (latest by date)](https://img.shields.io/github/v/release/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
6
- [![RubyGems Downloads](https://img.shields.io/gem/dt/fluent-plugin-protobuf-http?color=blue&style=flat-square)](https://rubygems.org/gems/fluent-plugin-protobuf-http)
6
+ [![RubyGems Downloads](https://img.shields.io/gem/dt/fluent-plugin-protobuf-http?style=flat-square)](https://rubygems.org/gems/fluent-plugin-protobuf-http)
7
+ [![Buy Me a Coffee](https://img.shields.io/badge/Support-Buy%20Me%20A%20Coffee-orange.svg?style=flat-square)](https://www.buymeacoffee.com/iamazeem)
7
8
 
8
- ![Lines of code](https://img.shields.io/tokei/lines/github/iamAzeem/fluent-plugin-protobuf-http?label=LOC&style=flat-square)
9
- ![GitHub code size in bytes](https://img.shields.io/github/languages/code-size/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
10
- ![GitHub repo size](https://img.shields.io/github/repo-size/iamAzeem/fluent-plugin-protobuf-http?style=flat-square)
9
+ ## Overview
11
10
 
12
- [Fluentd](https://fluentd.org/) HTTP input plugin for Protocol Buffers.
11
+ [Fluentd](https://fluentd.org/) HTTP input plugin for
12
+ [Protocol Buffers](https://github.com/protocolbuffers/protobuf).
13
13
 
14
14
  ## Features
15
15
 
16
- * Automatic compilation of `.proto` files located in `proto_dir`
17
- * Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
16
+ - Automatic compilation of `.proto` files located in `proto_dir`
17
+ - Incoming Format: Binary or JSON (`Content-Type`: `application/octet-stream` or
18
18
  `application/json`)
19
- * Outgoing Format: Binary or JSON
20
- * Single and Batch message support
21
- * TLS Support with `<transport>` section and `https://` URL protocol prefix.
19
+ - Outgoing Format: Binary or JSON
20
+ - Single and Batch message support
21
+ - TLS Support with `<transport>` section and `https://` URL protocol prefix.
22
22
 
23
23
  For more details on TLS configuration, see this official
24
24
  [example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
@@ -33,7 +33,7 @@ gem install fluent-plugin-protobuf-http
33
33
 
34
34
  ### Bundler
35
35
 
36
- Add following line to your Gemfile:
36
+ Add the following line to your Gemfile:
37
37
 
38
38
  ```ruby
39
39
  gem 'fluent-plugin-protobuf-http'
@@ -47,25 +47,25 @@ bundle
47
47
 
48
48
  ## Configuration
49
49
 
50
- * `bind` (string) (optional): The address to listen to.
51
- * Default: `0.0.0.0`
52
- * `port` (integer) (optional): The port to listen to.
53
- * Default: `8080`
54
- * `proto_dir` (string) (required): The directory path that contains the .proto files.
55
- * `in_mode` (enum) (optional): The mode of incoming (supported) events.
56
- * Modes: `binary`, `json`
57
- * Default: `binary`
58
- * `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
59
- * Modes: `binary`, `json`
60
- * Default: `binary`
61
- * `tag` (string) (required): The tag for the event.
50
+ - `bind` (string) (optional): The address to listen to.
51
+ - Default: `0.0.0.0`
52
+ - `port` (integer) (optional): The port to listen to.
53
+ - Default: `8080`
54
+ - `proto_dir` (string) (required): The directory path that contains the .proto files.
55
+ - `in_mode` (enum) (optional): The mode of incoming (supported) events.
56
+ - Modes: `binary`, `json`
57
+ - Default: `binary`
58
+ - `out_mode` (enum) (optional): The mode of outgoing (emitted) events.
59
+ - Modes: `binary`, `json`
60
+ - Default: `binary`
61
+ - `tag` (string) (required): The tag for the event.
62
62
 
63
63
  ### `<transport>` section (optional) (single)
64
64
 
65
- * `protocol` (enum) (optional):
66
- * Protocols: `tcp`, `tls`
67
- * Default: `tcp`
68
- * For more details, see this official configuration
65
+ - `protocol` (enum) (optional):
66
+ - Protocols: `tcp`, `tls`
67
+ - Default: `tcp`
68
+ - For more details, see this official configuration
69
69
  [example](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-server#configuration-example).
70
70
 
71
71
  ### Example
@@ -150,8 +150,8 @@ message Batch {
150
150
  }
151
151
  ```
152
152
 
153
- IMPORTANT: The `Batch` message type is part of `log.proto`, it is not a separate
154
- file! You can choose any name for a batch message type.
153
+ **IMPORTANT**: The `Batch` message type is part of `log.proto`, it is not a
154
+ separate file! You can choose any name for a batch message type.
155
155
 
156
156
  Here is the complete `log.proto` file:
157
157
 
@@ -263,9 +263,9 @@ For a simple use-case of incoming HTTP events and their routing to
263
263
  </match>
264
264
  ```
265
265
 
266
- The incoming binary messages will be transformed to JSON for further consumption.
266
+ The incoming binary messages will be converted to JSON for further consumption.
267
267
 
268
- ### Single Message
268
+ ### Single Message Use-case
269
269
 
270
270
  Test Parameters:
271
271
 
@@ -297,10 +297,10 @@ curl -X POST -H "Content-Type: application/octet-stream" \
297
297
  2020-06-09 18:53:47 +0500 [info]: #0 [protobuf_http_input] [S] {json} [127.0.0.1:41222, msgtype: service.logging.Log, size: 183 bytes]
298
298
  ```
299
299
 
300
- For sample Single message generation, see
301
- [this](https://github.com/iamAzeem/protobuf-log-sample).
300
+ For generating sample Single messages, see
301
+ https://github.com/iamAzeem/protobuf-log-sample.
302
302
 
303
- ### Batch Message
303
+ ### Batch Message Use-case
304
304
 
305
305
  Test Parameters:
306
306
 
@@ -363,16 +363,45 @@ $ curl -X POST -H "Content-Type: application/octet-stream" \
363
363
  ```
364
364
 
365
365
  For sample Batch message generation, see
366
- [this](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
366
+ [this gist](https://gist.github.com/iamAzeem/a8a24092132e1741a76956192f2104cc).
367
+
368
+ ## CI Workflow
369
+
370
+ The [CI workflow](ci..github/workflows/ci.yml) sets up the prerequisites. It
371
+ builds and installs the plugin, and then runs the automated tests.
372
+
373
+ To run tests locally, run:
374
+
375
+ ```shell
376
+ bundle exec rake test
377
+ ```
378
+
379
+ The [test](./test) directory contains the tests and the [input](./test/data)
380
+ files.
381
+
382
+ The code coverage is printed at the end using `simplecov`.
383
+
384
+ ## Known Issues
385
+
386
+ - This plugin internally uses the HTTP server plugin
387
+ [helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server)
388
+ which has higher precedence for `async-http` over `webrick`. But,
389
+ [`webrick`](https://github.com/ruby/webrick) is required to run this. In an
390
+ environment where both are installed, `async-http` is automatically selected
391
+ causing runtime issues. To make this work, you need to uninstall `async-http`
392
+ i.e. `gem uninstall async-http`. See issue
393
+ [#10](https://github.com/iamazeem/fluent-plugin-protobuf-http/issues/10) for
394
+ more details where this was identified in Docker containers where both gems
395
+ were already installed.
367
396
 
368
397
  ## Contribute
369
398
 
370
- - Fork the project.
399
+ - [Fork](https://github.com/iamazeem/fluent-plugin-protobuf-http/fork) the project.
371
400
  - Check out the latest `main` branch.
372
- - Create a feature or bugfix branch from `main`.
401
+ - Create a `feature` or `bugfix` branch from `main`.
373
402
  - Commit and push your changes.
374
403
  - Make sure to add and run tests locally: `bundle exec rake test`.
375
- - Run `rubocop` locally and fix all the lint warnings.
404
+ - Run [Rubocop](https://github.com/rubocop/rubocop) and fix the lint errors.
376
405
  - Submit the PR.
377
406
 
378
407
  ## License
data/fluent-plugin-protobuf-http.gemspec CHANGED
@@ -5,7 +5,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = 'fluent-plugin-protobuf-http'
8
- spec.version = '0.3.0'
8
+ spec.version = '0.3.1'
9
+ spec.date = '2023-09-16'
9
10
  spec.authors = ['Azeem Sajid']
10
11
  spec.email = ['azeem.sajid@gmail.com']
11
12
 
data/lib/fluent/plugin/in_protobuf_http.rb CHANGED
@@ -116,9 +116,10 @@ module Fluent
116
116
  log.debug("Extracting message types [#{compiled_proto}]...")
117
117
  msg_types = []
118
118
  File.foreach(compiled_proto) do |line|
119
- if line.lstrip.start_with?('add_message')
120
- msg_type = line[/"([^"]*)"/, 1] # regex: <add_message> 'msg_type' <do>
121
- msg_types.push(msg_type) unless msg_type.nil?
119
+ line.strip!
120
+ if line.include?('::Google::Protobuf::DescriptorPool.generated_pool.lookup') && line.end_with?('.msgclass')
121
+ extracted_msg_type = line[/"([^"]*)"/, 1].freeze
122
+ msg_types.push(extracted_msg_type) unless extracted_msg_type.nil?
122
123
  end
123
124
  end
124
125
 
metadata CHANGED
@@ -1,35 +1,35 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-protobuf-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Azeem Sajid
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-15 00:00:00.000000000 Z
11
+ date: 2023-09-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - "~>"
18
- - !ruby/object:Gem::Version
19
- version: '2.1'
20
17
  - - ">="
21
18
  - !ruby/object:Gem::Version
22
19
  version: 2.1.0
20
+ - - "~>"
21
+ - !ruby/object:Gem::Version
22
+ version: '2.1'
23
23
  type: :development
24
24
  prerelease: false
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
- - - "~>"
28
- - !ruby/object:Gem::Version
29
- version: '2.1'
30
27
  - - ">="
31
28
  - !ruby/object:Gem::Version
32
29
  version: 2.1.0
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '2.1'
33
33
  - !ruby/object:Gem::Dependency
34
34
  name: rake
35
35
  requirement: !ruby/object:Gem::Requirement
@@ -126,6 +126,7 @@ executables: []
126
126
  extensions: []
127
127
  extra_rdoc_files: []
128
128
  files:
129
+ - ".github/dependabot.yml"
129
130
  - ".github/workflows/ci.yml"
130
131
  - ".gitignore"
131
132
  - ".rubocop.yml"
@@ -160,8 +161,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
160
161
  - !ruby/object:Gem::Version
161
162
  version: '0'
162
163
  requirements: []
163
- rubyforge_project:
164
- rubygems_version: 2.7.6
164
+ rubygems_version: 3.0.9
165
165
  signing_key:
166
166
  specification_version: 4
167
167
  summary: fluentd HTTP Input Plugin for Protocol Buffers