fluent-plugin-bcdb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 98b39e5a8b2ac790a5e7bc53baa1381756ea0f76a65f88693aca421aecbcfeb3
+  data.tar.gz: ff504db40716d7f4a3486499620f2cf07ce3c36d16a3dac7e79adae8688b4377
+SHA512:
+  metadata.gz: a7dd58c903129d67e9d7d017a1a583217aef0faef99877b9b216b693e7d0069a39ad4d4e1afbb04c8dd559c3514d1d069b81d8144cd201b5b118c5306126d843
+  data.tar.gz: 11b7985b329d822face35e003aa30674cc8aeb43f2cff8646cea95c2f4885e9b8ee30e00c60ff7d19e8466b8072c8aaacc7192cc60ed1e48777ca25b5078fbe2

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at TODO: Write your email address. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Dockerfile

@@ -0,0 +1,28 @@
+#
+# Copyright ©2019. MODEX (Gibraltar) LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM fluent/fluentd
+
+USER root
+
+RUN apk add --no-cache --update --virtual .build-deps \
+        sudo build-base ruby-dev git bash
+COPY ./ /fluent-plugin-bcdb/
+RUN gem install bundler
+WORKDIR /fluent-plugin-bcdb/
+RUN bundle install
+RUN bundle exec rake install
+# cutomize following instruction as you wish
+ # USER root

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fluent-plugin-bcdb.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,51 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-bcdb (0.1.0)
+      fluentd (>= 0.14.22, < 2)
+      yajl-ruby (~> 1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    concurrent-ruby (1.1.5)
+    cool.io (1.5.4)
+    dig_rb (1.0.1)
+    fluentd (1.7.2)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.2.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 2.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    msgpack (1.3.1)
+    power_assert (1.1.5)
+    rake (10.5.0)
+    serverengine (2.1.1)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.3)
+    test-unit (3.3.3)
+      power_assert
+    tzinfo (2.0.0)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2019.3)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  fluent-plugin-bcdb!
+  rake (~> 10.0)
+  test-unit (>= 3.1.0)
+
+BUNDLED WITH
+   2.0.2

data/LICENSE

@@ -0,0 +1,204 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+
+
+   Copyright [2019] [MODEX (Gibraltar) LIMITED]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,78 @@
+# Fluentd Modex BCDB connector
+
+fluentd-bcdb-connector enables immutable logs using [Modex BCDB](https://bcdb.modex.tech). Main features: 
+
+* Bulk insert of records
+* Buffered insert of records
+* Request compression
+
+## Installation
+
+> **NOTE:** First see [here](https://bcdb.modex.tech/docs/#/install/install) the installation guide for Modex BCDB
+---
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-bcdb'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-bcdb
+
+## Usage
+
+## Configuration options
+
+> **NOTE:** this is a configuration example using Modex BCDB [sandbox](https://bcdb.modex.tech)
+
+    <match>
+        @type bcdb
+        base_url        https://bcdb.modex.tech/node-03/services/core/v1/api
+        auth_url        https://bcdb.modex.tech/oauth/token
+        ssl_no_verify   false  # default: false
+        rate_limit_msec 100    # default: 0 = no rate limiting
+        raise_on_error  true  # default: true
+        recoverable_status_codes 403, 400, 503, 500 # default: 503
+        cacert_file     /etc/ssl/endpoint1.cert # default: ''
+        client_cert_path /path/to/client_cert.crt # default: ''
+        private_key_path /path/to/private_key.key # default: ''
+        private_key_passphrase yourpassphrase # default: ''
+        username        bcdb.admin@modex.tech # default: ''
+        password        BCDBDemo2019! # default: '', secret: true
+        client_id       0x01 # BDCB client_id
+        client_secret   0x000001 # BDCB client_secret
+        buffered        false   # default: false. Switch non-buffered/buffered mode
+        bulk_request    false   # default: false. Send events as application/x-ndjson
+        compress_request false  # default: false. Send compressed events
+    </match>
+
+
+## Usage notes
+
+If you want to retry failed requests, specify the appropriate `recoverable_status_codes` parameter.
+
+To send events with bulk_request, you need to specify bulk_request as true. Note that when this parameter is true, Fluentd will always send events as application/x-ndjson. Currently, application/x-ndjson is the only supported MIME type for bulk_request.
+
+## Development
+
+After you checkout the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+[Bug reports](https://github.com/modex-bcdb/fluentd-bcdb-connector/issues) and [pull requests](https://github.com/modex-bcdb/fluentd-bcdb-connector/pulls) are welcomed on GitHub. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [Apache License](https://opensource.org/licenses/Apache-2.0).
+
+## Code of Conduct
+
+Everyone interacting in the Fluent::Plugin::Bcdb project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/modex-bcdb/fluentd-bcdb-connector/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/bcdb"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/dev/fluent.conf

@@ -0,0 +1,37 @@
+<source>
+  @type  http
+  @id    input1
+  @label @mainstream
+  @log_level trace
+  port  24224
+</source>
+
+<system>
+  # equal to -qq option
+  log_level trace
+</system>
+
+<label @mainstream>
+    <match **>
+      @type bcdb
+      base_url        "http://<ip>:<port>/services/core/v1/api" # Api endpoint like https://bcdb.modex.tech/node-03/services/core/v1/api
+      auth_url        "http://<ip>:<port>/token" # OAuth token endpoint like https://bcdb.modex.tech/oauth/token
+      bcdb_entity     "log"  # default: logline
+      ssl_no_verify   false  # default: false
+      rate_limit_msec 100    # default: 0 = no rate limiting
+      raise_on_error  true   # default: true
+      recoverable_status_codes 5000, 400 # default: 503
+      cacert_file     /etc/ssl/endpoint1.cert   # default: ''
+      client_cert_path /path/to/client_cert.crt # default: ''
+      private_key_path /path/to/private_key.key # default: ''
+      private_key_passphrase yourpassphrase     # default: ''
+      username        <username> # ex. bcdb.admin@modex.tech default: ''
+      password        <password> # ex. admin default: '', secret: true
+      client_id       0x01      # default: ''. BDCB client_id
+      client_secret   0x000001  # default: ''. BDCB client_secret
+      buffered        true     # default: false. Switch non-buffered/buffered mode
+      bulk_request    true     # default: false. Send events as application/x-ndjson
+      compress_request false    # default: false. Send compressed events
+      @log_level trace
+    </match>
+</label>

data/docker-compose.yaml

@@ -0,0 +1,38 @@
+#
+# Copyright ©2019. MODEX (Gibraltar) LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: '2'
+services:
+  web:
+    image: httpd
+    ports:
+      - "81:80"
+    links:
+      - fluentd
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: localhost:24224
+        tag: httpd.access
+  fluentd:
+    build: ./
+    volumes:
+      - ./dev/fluent.conf:/fluentd/etc/fluent.conf
+      - ./:/plugins
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    tty: true
+    command: ["/usr/bin/ruby", "-Eascii-8bit:ascii-8bit", "/usr/bin/fluentd", "-c","/fluentd/etc/fluent.conf"]

data/fluent-plugin-bcdb.gemspec

@@ -0,0 +1,48 @@
+#
+# Copyright ©2019. MODEX (Gibraltar) LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "fluent/plugin/bcdb/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-bcdb"
+  spec.version       = Fluent::Plugin::Bcdb::VERSION
+  spec.authors       = ["Silviu Panaite"]
+  spec.email         = ["silviu@messbusters.org"]
+
+  spec.summary       = %q{Fluent output plugin to Modex Blockchain Database}
+  spec.homepage      = "https://github.com/messbusters/fluent-plugin-bcdb"
+  spec.license       = "APACHE"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
+  spec.metadata["changelog_uri"] = "https://github.com/messbusters/fluent-plugin-bcdb"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "yajl-ruby", "~> 1.0"
+  spec.add_dependency "fluentd", [">= 0.14.22", "< 2"]
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+end

data/lib/fluent/plugin/bcdb/version.rb

@@ -0,0 +1,7 @@
+module Fluent
+  module Plugin
+    module Bcdb
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/fluent/plugin/out_bcdb.rb

@@ -0,0 +1,459 @@
+#
+# Copyright ©2019. MODEX (Gibraltar) LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'net/http'
+require 'uri'
+require 'yajl'
+require 'fluent/plugin/output'
+require "fluent/plugin/bcdb/version"
+require 'tempfile'
+require 'openssl'
+require 'zlib'
+
+class BcdbOut < Fluent::Plugin::Output
+  Fluent::Plugin.register_output('bcdb', self)
+
+  class RecoverableResponse < StandardError; end
+
+  helpers :compat_parameters, :formatter
+
+  DEFAULT_BUFFER_TYPE = "memory"
+  DEFAULT_FORMATTER = "json"
+
+  def initialize
+    super
+  end
+
+  # BCDB data endpoint
+  config_param :base_url, :string
+
+  # BCDB Auth endpoint
+  config_param :auth_url, :string
+
+  # BCDB database entity model name
+  config_param :bcdb_entity, :string, :default => 'loglines'
+
+  # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
+  config_param :ssl_no_verify, :bool, :default => false
+
+  # HTTP method
+  config_param :http_method, :enum, list: [:get, :put, :post, :delete], :default => :post
+
+  # form | json | text | raw
+  config_param :serializer, :enum, list: [:json, :form, :text, :raw], :default => :json
+
+  # Simple rate limiting: ignore any records within `rate_limit_msec`
+  # since the last one.
+  config_param :rate_limit_msec, :integer, :default => 0
+
+  # Raise errors that were rescued during HTTP requests?
+  config_param :raise_on_error, :bool, :default => true
+
+  # Specify recoverable error codes
+  config_param :recoverable_status_codes, :array, value_type: :integer, default: [503]
+
+  # ca file to use for https request
+  config_param :cacert_file, :string, :default => ''
+
+  # specify client sertificate
+  config_param :client_cert_path, :string, :default => ''
+
+  # specify private key path
+  config_param :private_key_path, :string, :default => ''
+
+  # specify private key passphrase
+  config_param :private_key_passphrase, :string, :default => '', :secret => true
+
+  # custom headers
+  config_param :custom_headers, :hash, :default => nil
+
+  # 'none' | 'basic' | 'jwt' | 'bearer'
+  config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer, :oauth],  :default => :oauth
+  config_param :username, :string, :default => ''
+  config_param :password, :string, :default => '', :secret => true
+  config_param :client_id, :string, :default => ''
+  config_param :client_secret, :string, :default => '', :secret => true
+  config_param :grant_type, :enum, list: [:password, :authorization_code], :default => :password
+  config_param :token, :string, :default => ''
+  # Switch non-buffered/buffered plugin
+  config_param :buffered, :bool, :default => false
+  config_param :bulk_request, :bool, :default => false
+  # Compress with gzip except for form serializer
+  config_param :compress_request, :bool, :default => false
+
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
+    config_set_default :chunk_keys, ['tag']
+  end
+
+  config_section :format do
+    config_set_default :@type, DEFAULT_FORMATTER
+  end
+
+  def configure(conf)
+    compat_parameters_convert(conf, :buffer, :formatter)
+    super
+    @create_schema_url = "#{@base_url}" + "/catalog/_JsonSchema/" + "#{@bcdb_entity}"
+    if @bulk_request
+        @base_url = "#{@base_url}" + "/data/bulk/" + "#{@bcdb_entity}"
+    else
+        @base_url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
+    end
+
+    bcdb_authorise() if @authentication == :oauth
+
+    @ssl_verify_mode = if @ssl_no_verify
+                         OpenSSL::SSL::VERIFY_NONE
+                       else
+                         OpenSSL::SSL::VERIFY_PEER
+                       end
+
+    @ca_file = @cacert_file
+    @last_request_time = nil
+    raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered
+
+    if @formatter_config = conf.elements('format').first
+      @formatter = formatter_create
+    end
+
+    if @bulk_request
+      class << self
+        alias_method :format, :bulk_request_format
+      end
+      @formatter = formatter_create(type: :json)
+      @serializer = :x_ndjson # secret settings for bulk_request
+    else
+      class << self
+        alias_method :format, :split_request_format
+      end
+    end
+  end
+
+  def bcdb_authorise()
+      auth_uri = URI.parse(@auth_url)
+      auth_data = {
+          :username => @username,
+          :password => @password,
+          :client_id => @client_id,
+          :client_secret => @client_secret,
+          :grant_type => @grant_type
+      }
+      status = true
+      unless @token_oauth || (@expires_token && Time.now.utc > @expires_token)
+          https= Net::HTTP.new(auth_uri.host,auth_uri.port)
+          https.use_ssl = auth_uri.scheme == 'https'
+
+          request = Net::HTTP::Post.new(auth_uri.path)
+          request.set_form_data(auth_data)
+          request['Content-Type'] = "application/x-www-form-urlencoded"
+          resp = https.request(request)
+          log.debug("#{resp.body}")
+          bcdb_response = JSON.parse(resp.body)
+          if bcdb_response["code"] == 5000
+              status = false
+              log.error("Authentification failed please check your credentials")
+          else
+              @token_oauth = bcdb_response['access_token']
+              @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
+          end
+      end
+      return status
+  end
+
+  def bcdb_update_schema(data, cached_keys=false)
+      schema_uri = URI.parse(@create_schema_url)
+      schema_properties = {}
+      data.each do |key|
+          log.debug("KEY #{key.inspect}")
+          schema_properties["#{key}"] = {
+              :"$id" => "/properties/#{schema_properties["#{key}"]}",
+              :type => "string",
+              :title => "The #{schema_properties["#{key}"]} Schema"
+          }
+      end
+      schema_data = {
+          :type => "object",
+          :"$id" => @bcdb_entity,
+          :"$schema" => "http://json-schema.org/draft-07/schema#",
+          :title => "The Root Schema",
+          :properties => schema_properties,
+          :autoId => true
+      }
+      body = JSON(schema_data)
+
+      if cached_keys
+          request = bcdb_url(schema_uri,'put', body)
+      else
+          request = bcdb_url(schema_uri,'post',body)
+          if JSON.parse(request.body)["code"] == 4009
+              request = bcdb_url(schema_uri,'put', body)
+          end
+      end
+     log.debug("UPDATE SCHEMA: #{body}")
+
+     log.debug("UPDATE SCHEMA RESPONSE: #{request.body}")
+     return data, true
+  end
+  def bcdb_url(uri,type,body)
+      bcdb_request = Net::HTTP.new(uri.host,uri.port)
+      bcdb_request.use_ssl = uri.scheme == 'https'
+      case type
+      when 'post'
+          request = Net::HTTP::Post.new(uri.path)
+      when 'put'
+          request = Net::HTTP::Put.new(uri.path)
+      end
+      request.body = body
+      request['Content-Type'] = "application/json"
+      request['authorization'] = "Bearer #{@token_oauth}"
+      response = bcdb_request.request(request)
+      return response
+  end
+
+  def start
+    super
+  end
+
+  def shutdown
+    super
+  end
+
+  def format_url(tag, time, record)
+    @base_url
+  end
+
+  def set_body(req, tag, time, record)
+    if @serializer == :json
+      set_json_body(req, record)
+    elsif @serializer == :text
+      set_text_body(req, record)
+    elsif @serializer == :raw
+      set_raw_body(req, record)
+    elsif @serializer == :x_ndjson
+      set_bulk_body(req, record)
+    else
+      req.set_form_data(record)
+    end
+    req
+  end
+
+  def set_header(req, tag, time, record)
+    if @custom_headers
+      @custom_headers.each do |k,v|
+        req[k] = v
+      end
+      req
+    else
+      req
+    end
+  end
+
+  def compress_body(req, data)
+    return unless @compress_request
+    gz = Zlib::GzipWriter.new(StringIO.new)
+    gz << data
+
+    req['Content-Encoding'] = "gzip"
+    req.body = gz.close.string
+  end
+
+  def set_json_body(req, data)
+    bcdb_authorise()
+    unless @cached_keys && @keys.sort == data.keys.sort
+        @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
+    end
+    # data = { :records => [data] } if @bulk_request
+    req.body = Yajl.dump(data)
+    req['Content-Type'] = "application/json"
+    compress_body(req, req.body)
+  end
+
+  def set_text_body(req, data)
+    req.body = data["message"]
+    req['Content-Type'] = 'text/plain'
+    compress_body(req, req.body)
+  end
+
+  def set_raw_body(req, data)
+    req.body = data.to_s
+    req['Content-Type'] = 'application/octet-stream'
+    compress_body(req, req.body)
+  end
+
+  def set_bulk_body(req, data)
+    bcdb_authorise()
+    if data.is_a? String
+        flat_keys = []
+        bcdb_data = data.split("\n").map{ |x| JSON.parse(x) }
+        bcdb_data.each do |data|
+            flat_keys = flat_keys + data.keys
+        end
+        flat_keys.uniq!
+        unless @cached_keys && @keys.sort == flat_keys.sort
+            @keys, @cached_keys = bcdb_update_schema(flat_keys, @cached_keys)
+        end
+        data = { :records => bcdb_data }
+        @base_url = "#{@base_url_}" + "/data/bulk/" + "#{@bcdb_entity}"
+    else
+        log.debug("DATA: #{data.inspect}")
+        unless @cached_keys && @keys.sort == data.keys.sort
+            @keys, @cached_keys = bcdb_update_schema(data.keys, @cached_keys)
+        end
+        data = { :records => [data] }
+    end
+    req.body = Yajl.dump(data)
+    # req['Content-Type'] = 'application/x-ndjson'
+    req['Content-Type'] = 'application/json'
+    compress_body(req, req.body)
+  end
+
+  def create_request(tag, time, record)
+    url = format_url(tag, time, record)
+    uri = URI.parse(url)
+    req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.request_uri)
+    set_body(req, tag, time, record)
+    set_header(req, tag, time, record)
+    log.trace("CREATE REQUEST: #{req}, #{uri}")
+    return req, uri
+  end
+
+  def http_opts(uri)
+      opts = {
+        :use_ssl => uri.scheme == 'https'
+      }
+      opts[:verify_mode] = @ssl_verify_mode if opts[:use_ssl]
+      opts[:ca_file] = File.join(@ca_file) if File.file?(@ca_file)
+      opts[:cert] = OpenSSL::X509::Certificate.new(File.read(@client_cert_path)) if File.file?(@client_cert_path)
+      opts[:key] = OpenSSL::PKey.read(File.read(@private_key_path), @private_key_passphrase) if File.file?(@private_key_path)
+      opts
+  end
+
+  def proxies
+    ENV['HTTPS_PROXY'] || ENV['HTTP_PROXY'] || ENV['http_proxy'] || ENV['https_proxy']
+  end
+
+  def send_request(req, uri)
+    is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+    if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+      log.info('Dropped request due to rate limiting')
+      return
+    end
+
+    res = nil
+
+    begin
+      if @authentication == :basic
+        req.basic_auth(@username, @password)
+      elsif @authentication == :bearer
+        req['authorization'] = "bearer #{@token}"
+      elsif @authentication == :jwt
+        req['authorization'] = "jwt #{@token}"
+      elsif @authentication == :oauth
+          req['authorization'] = "Bearer #{@token_oauth}"
+      end
+      @last_request_time = Time.now.to_f
+
+      if proxy = proxies
+        proxy_uri = URI.parse(proxy)
+
+        res = Net::HTTP.start(uri.host, uri.port,
+                              proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password,
+                              **http_opts(uri)) {|http| http.request(req) }
+      else
+        res = Net::HTTP.start(uri.host, uri.port, **http_opts(uri)) {|http| http.request(req) }
+        log.debug("REQUEST BODY: #{req.body}")
+        log.debug("RESPONSE BODY: #{res.body}")
+      end
+    rescue => e # rescue all StandardErrors
+      # server didn't respond
+      log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+      raise e if @raise_on_error
+    else
+       unless res and res.is_a?(Net::HTTPSuccess)
+          res_summary = if res
+                           "#{res.code} #{res.message} #{res.body}"
+                        else
+                           "res=nil"
+                        end
+          if @recoverable_status_codes.include?(res.code.to_i)
+            raise RecoverableResponse, res_summary
+          else
+            log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+          end
+       end #end unless
+    end # end begin
+  end # end send_request
+
+  def handle_record(tag, time, record)
+    if @formatter_config
+      record = @formatter.format(tag, time, record)
+    end
+    req, uri = create_request(tag, time, record)
+    send_request(req, uri)
+  end
+
+  def handle_records(tag, time, chunk)
+    req, uri = create_request(tag, time, chunk.read)
+    send_request(req, uri)
+  end
+
+  def prefer_buffered_processing
+    @buffered
+  end
+
+  def format(tag, time, record)
+    # For safety.
+  end
+
+  def split_request_format(tag, time, record)
+    [time, record].to_msgpack
+  end
+
+  def bulk_request_format(tag, time, record)
+    @formatter.format(tag, time, record)
+  end
+
+  def formatted_to_msgpack_binary?
+    if @bulk_request
+      false
+    else
+      true
+    end
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def process(tag, es)
+    log.trace("TRACE PROCESS: #{tag}, #{es}")
+    es.each do |time, record|
+      handle_record(tag, time, record)
+    end
+  end
+
+  def write(chunk)
+    tag = chunk.metadata.tag
+    @base_url = extract_placeholders(@base_url, chunk)
+    if @bulk_request
+      time = Fluent::Engine.now
+      handle_records(tag, time, chunk)
+    else
+      chunk.msgpack_each do |time, record|
+        handle_record(tag, time, record)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-bcdb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Silviu Panaite
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: 
+email:
+- silviu@messbusters.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dev/fluent.conf
+- docker-compose.yaml
+- fluent-plugin-bcdb.gemspec
+- lib/fluent/plugin/bcdb/version.rb
+- lib/fluent/plugin/out_bcdb.rb
+homepage: https://github.com/messbusters/fluent-plugin-bcdb
+licenses:
+- APACHE
+metadata:
+  homepage_uri: https://github.com/messbusters/fluent-plugin-bcdb
+  source_code_uri: https://github.com/messbusters/fluent-plugin-bcdb
+  changelog_uri: https://github.com/messbusters/fluent-plugin-bcdb
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6.2
+signing_key: 
+specification_version: 4
+summary: Fluent output plugin to Modex Blockchain Database
+test_files: []