fluent-plugin-http_forward 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d17721c462207dd57050c8f0549ca074ad1ca630
+  data.tar.gz: 2232c90bdb2fee35784415dcfa506c0707de10f2
+SHA512:
+  metadata.gz: 9e31ea459957415c2f87a9097c5ca052b6fb21d558344e0384b861c53e05b297f54d261e8516a0b9dc8f7ac4e048e6205d91fd7f0c69e81e167fb0c71fe410fd
+  data.tar.gz: f42b9ea013bd072c3150f1f84f251d4cc69127c6ce5ca789f9ee5fc0f67973bc796788d4283bfe0d0b2c22372505fd0b70f92ad4813725d076133bd93f33399d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem

data/CHANGELOG.txt

@@ -0,0 +1,5 @@
+# Changelog
+
+v0.1.0
+---------------
+* Initial release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-http_batch.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# Fluent::Plugin::HttpForward
+
+This gem provides a buffered HTTP output plugin for Fluentd which also optionally supports authentication.
+
+## Requirements
+
+* Fluetnd v0.14+
+
+## Installation
+
+```ruby
+gem 'fluent-plugin-http_forward'
+```
+
+## Configuration Options
+
+```
+<match **>
+  @type http_forward
+  url   http://remote.example.com/%{tag}
+  verb  post
+  content_type  application/json # optional override
+  <authentication>
+    method    basic
+    username  user
+    password  pass
+  </authentication>
+  <format>
+    @type json
+  </format>
+  <buffer tag>                  # if buffering, specify tag for in_http compat
+    chunk_limit_size 8388608    # optional, maximum chunk size, default 8MB
+    total_limit_size 536760912  # optional, maximum buffer size, default 512M
+    chunk_records_limit 10      # optional, maximum records per chunk
+    flush_interval 60           # optional, maximum time between flush
+    flush_thread_count 1        # optional, async threads to flush buffer
+  </buffer>
+```
+
+## Status
+
+At present, only the JSON format has been tested. There's an optional msgpack format available that is as of yet untested because in_http does not currently support it. 
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fluent/plugin/http_batch"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/data.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for x in `seq 1 10000`; do
+  echo "RPUSH test \"{ \\\"message\\\": \\\"${x}\\\" }\""
+done | redis-cli --pipe
+

data/examples/standalone.conf

@@ -0,0 +1,36 @@
+<source>
+  @type redis_list_poller
+  tag	redis.content
+  host	127.0.0.1
+  key	test
+  batch_size 1
+  poll_interval 0.1
+  <parse>
+    @type json
+  </parse>
+</source>
+
+<source>
+  @type http
+  bind 127.0.0.1
+  format json
+</source>
+
+<match redis.content>
+  @type http_forward
+  url   http://127.0.0.1:9880/dummy_tag
+  flush_thread_count 2
+  <format>
+    @type json
+  </format>
+  <buffer tag>
+    @type memory
+    chunk_records_limit 10
+  </buffer>
+   flush_interval 1
+</match>
+
+<match dummy_tag>
+  @type flowcounter_simple
+  unit second
+</match>

data/flunt-plugin-http_forward.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-http_forward"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Jonathan Serafini"]
+  spec.email         = ["jonathan@serafini.ca"]
+
+  spec.summary       = %q{A buffered HTTP batching output for Fluentd}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/JonathanSerafini/fluent-plugin-http_forward"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = []
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
+  spec.add_runtime_dependency "http", [">= 2.0.0", "< 3.0.0"]
+end

data/lib/fluent/plugin/out_http_forward.rb

@@ -0,0 +1,181 @@
+require 'fluent/plugin/output'
+require 'fluent/plugin_mixin/chunk_json_streamer'
+
+module Fluent
+  module Plugin
+    class HttpForwardOutput < Output
+      Fluent::Plugin.register_output('http_forward', self)
+      helpers :timer
+
+      # http target parameters
+      # - url: the URL to upload to, ex.: http://iamateacup:8080/path/%{tag}
+      #        the URI may contain the %{tag} pattern which will be replaced
+      # - verb: the HTTP verb to use 
+      config_param :url,    :string
+      config_param :verb,   :string,  :default => :post,
+                                      :list => %w(put post)
+
+      # an optional Content-Type header override, this may be unset and will be
+      # automatically detected based on the serializing format
+      config_param :content_type, :string, :default => nil
+
+      # http authentication parameters
+      config_section :authentication, :init => true, :multi => false do
+        config_param :method,   :string, :default => nil, :list => %(basic)
+        config_param :username, :string, :default => nil
+        config_param :password, :string, :default => nil, :secret => true
+      end
+
+      # output serializing format
+      config_section :format, :init => true, :multi => false do
+        config_param :@type, :string, :default => "json",
+                                      :list => %w(msgpack json)
+      end
+
+      # Initialize new output plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def initialize
+        super
+        require 'http'
+      end
+
+      # Initialize attributes and parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure(config)
+        super
+
+        configure_params(config)
+      end
+
+      # Configure plugin parameters
+      # @since 0.1.0
+      # @return [NilClass]
+      def configure_params(config)
+        unless config.key?("content_type")
+          @content_type = case @format['@type']
+            when 'json' then 'application/json'
+            when 'msgpack' then 'application/x-msgpack'
+            else nil
+            end
+        end
+      end
+
+      # Prepare the plugin event loop
+      # @since 0.1.0
+      # @return [NilClass]
+      def start
+        super
+        start_connection
+      end
+
+      # Prepare the HTTP client object which provides a baseline for future
+      # request objects. 
+      # @since 0.1.0
+      # @return [HTTP::Client]
+      def start_connection
+        @connection = HTTP::Client.new(
+          :headers => @headers
+        )
+
+        if @authentication['method']
+          @connection = @connection.basic_auth(
+            :user => @authentication['username'],
+            :pass => @authentication['password']
+          )
+        end
+     
+        if @content_type
+          @connection = @connection.headers(
+            'Content-Type' => @content_type
+          )
+        end
+
+        @connection
+      end
+
+      # Tear down the plugin
+      # @since 0.1.0
+      # @return [NilClass]
+      def shutdown
+        @connection.close
+        super
+      end
+
+      # Enforce the usage of the MsgPack streaming method internally so as to
+      # ensure that we have a consistent buffering mechanism.
+      # @since 0.1.0
+      # @return [NilClass]
+      def execute_chunking(tag, es, enqueue: false)
+        return handle_stream_with_standard_format(tag, es, enqueue: enqueue)
+      end
+
+      # Format the URL for a given request, and optionally replace the tag
+      # placerholder in the URI.
+      # @since 0.1.0
+      # @return [String] the url 
+      def format_url(tag, time, data)
+        @url.sub(/%{tag}/, tag)
+      end
+
+      # Format the Headers for a given request.
+      # @since 0.1.0
+      # @return [Hash] the header hash
+      def format_headers(tag, time, data)
+        @headers
+      end
+
+      # Format the Body for a given request depending on the Serializing
+      # format. 
+      # @since 0.1.0
+      # @return [String] the request body
+      def format_body(tag, time, chunk)
+        case @format["@type"]
+        when "json"
+          chunk.extend Fluent::PluginMixin::ChunkJsonStreamer
+          serializer_proc = chunk.method(:to_json_stream)
+        else
+          serializer_proc = chunk.method(:to_msgpack_stream)
+        end
+
+        body = serializer_proc.call
+      end
+
+      # Upload the request body to the remote end point
+      # @since 0.1.0
+      # @return [Response] the HTTP response object
+      def upload(tag, time, content)
+        headers = format_headers(tag, nil, content)
+        url = format_url(tag, nil, content)
+
+        @connection.request(@verb, url, body: content, headers: headers)
+      end
+
+      # Process a chunk of records synchronously, committing successful 
+      # uploads and re-queueing failures.
+      # @since 0.1.0
+      # @return [Nil]
+      def write(chunk)
+        tag = chunk.metadata.tag
+        content = format_body(tag, nil, chunk)
+
+        response = upload(tag, nil, content)
+
+        if response.code <= 299
+          commit_write(chunk.unique_id)
+        else
+          @log.warn "failed to flush buffer", code: response.code
+        end
+      end
+
+      # Process a chunk of records asynchronously, committing successful 
+      # uploads and re-queueing failures.
+      # @since 0.1.0
+      # @return [Nil]
+      def try_write(chunk)
+        write(chunk)
+      end
+    end
+  end
+end

data/lib/fluent/plugin_mixin/chunk_json_streamer.rb

@@ -0,0 +1,38 @@
+module Fluent
+  module PluginMixin
+    module ChunkJsonStreamer
+      # Iterate over records in chunk
+      # @since 0.1.0
+      # @returns [Array] an array of record objects
+      def records
+        records = []
+        each do |time, record|
+          records << record
+        end
+        records
+      end
+
+      # Serialize the chunk to a Json object
+      # @since 0.1.0
+      # @returns [String] the serialized object
+      def to_json_stream
+        json_serializer_proc.call(records)
+      end
+
+      private
+
+      # Load relevant Json modules and cache the serializing method
+      # @since 0.1.0
+      # @returns [Method] a proc which serializes to json
+      def json_serializer_proc
+        @json_packer ||=  begin
+                            require 'oj'
+                            Oj.default_options = { mode: :compat }
+                            Oj.method(:dump)
+                          rescue LoadError
+                            Yajl.method(:dump)
+                          end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,125 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-http_forward
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jonathan Serafini
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: A buffered HTTP batching output for Fluentd
+email:
+- jonathan@serafini.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CHANGELOG.txt
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- examples/data.sh
+- examples/standalone.conf
+- flunt-plugin-http_forward.gemspec
+- lib/fluent/plugin/out_http_forward.rb
+- lib/fluent/plugin_mixin/chunk_json_streamer.rb
+homepage: https://github.com/JonathanSerafini/fluent-plugin-http_forward
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: A buffered HTTP batching output for Fluentd
+test_files: []
+has_rdoc: