fluent-plugin-grassland 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3355444d5dbd8e2b8959c60beb289eb0d94e960d
+  data.tar.gz: f136db3ac51462deeb2417b1a01ed9128562aa38
+SHA512:
+  metadata.gz: 7abb4001d45fd3a37ff0fa2fcd825c78b0cd1c04d5534632874715884f18e029deadd29a6045fec3fb0acb6d6f0cc0b8f886dab2c6956c253316d714580d0f74
+  data.tar.gz: 96d60b8872a1f44f35544b93cc0deb9f3b923a7e367239179d95e0fbf3b12fb6103aaa5a08432bcd341fa99e69c33875f5f868f6f7f516e145cd293f6207a0fa

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis-alt.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Genki Sugawara
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# fluent-plugin-grassland
+
+Output filter plugin for Grassland
+
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis-alt.png)](http://badge.fury.io/rb/fluent-plugin-kinesis-alt)
+
+## Notice
+
+maybe write after.
+
+## Installation
+* It still not working
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-grassland'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-grassland
+
+## Usage
+
+### Configuration
+
+```
+<match kinesis.**>
+  type grassland
+  key xxxxxxxxxxxxxxxx
+  flush_interval 3
+</match>
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-grassland.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-grassland'
+  spec.version       = '0.0.1'
+  spec.authors       = ['Ripplation Inc.']
+  # spec.email         = ['xxxxxx@ripplation.co.jp']
+  spec.description   = 'Output filter plugin for Grassland'
+  spec.summary       = 'Output filter plugin for Grassland'
+  spec.homepage      = 'https://bitbucket.org/winebarrel/fluent-plugin-kinesis-alt'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'fluentd'
+  spec.add_dependency 'aws-sdk', '~> 1.38'
+  spec.add_dependency 'json'
+  # spec.add_development_dependency 'bundler', '~> 1.3'
+  # spec.add_development_dependency 'rake'
+end

data/lib/fluent/plugin/out_grassland.rb

@@ -0,0 +1,147 @@
+module Fluent
+  class GrasslandOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('grassland', self)
+
+    attr_accessor :apiuri, :stream_name, :access_key_id, :secret_access_key, :region
+
+    def initialize
+      super
+      require 'aws-sdk'
+      require 'base64'
+      require 'json'
+      require 'logger'
+      require 'net/http'
+      require 'uri'
+      @apiuri = 'https://s3.amazonaws.com/apicre/credential.json'
+    end
+
+    config_param :key,   :string, :default => nil
+    config_param :debug, :bool, :default => false
+
+    def configure(conf)
+      super
+
+      [:key].each do |name|
+        unless self.instance_variable_get("@#{name}")
+          raise ConfigError, "'#{name}' is required"
+        end
+      end
+    end
+
+    def start
+      super
+      setCredential
+      configure_aws
+      AWS.kinesis.client.put_record({
+        :stream_name   => @stream_name,
+        :data          => "test",
+        :partition_key => "#{rand(999)}"
+      })
+    end
+
+    def shutdown
+      super
+    end
+
+    def setCredential()
+      credential = get_json(@apiuri)
+      @stream_name = credential['streamName']
+      @access_key_id = credential['accessKeyId']
+      @secret_access_key = credential['secretAccessKey']
+      @region = credential['region']
+    end
+
+    def get_json(location, limit = 3)
+      raise ArgumentError, 'too many HTTP redirects' if limit == 0
+      uri = URI.parse(location)
+      begin
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+          http.open_timeout = 5
+          http.read_timeout = 10
+          http.get(uri.request_uri)
+        end
+        case response
+        when Net::HTTPSuccess
+          json = response.body
+          JSON.parse(json)
+        when Net::HTTPRedirection
+          location = response['location']
+          warn "redirected to #{location}"
+          get_json(location, limit - 1)
+        else
+          puts [uri.to_s, response.value].join(" : ")
+          # handle error
+        end
+      rescue => e
+        puts [uri.to_s, e.class, e].join(" : ")
+        # handle error
+      end
+    end
+
+    def format(tag, time, record)
+      # print(record)
+      ['cid', 'dt', 'uid', 'pt', 'd'].each do |key|
+        unless record.has_key?(key)
+          puts "input data error: '#{key}' is required"
+          return ""
+        end
+      end
+
+      record['pk'] = record['cid'] + record['dt']
+      return "#{record.to_json},"
+    end
+
+    def write(chunk)
+      buf = chunk.read
+      dataList = JSON.parse("[#{buf.chop}]")
+      putBuf = ""
+      bufList = {}
+
+      dataList.each do |data|
+        if bufList[":#{data['pk']}"] == nil then
+          bufList[":#{data['pk']}"] = "#{data.to_json},"
+        else
+          bufList[":#{data['pk']}"] += "#{data.to_json},"
+        end
+        if bufList[":#{data['pk']}"].bytesize >= 30720 then
+          AWS.kinesis.client.put_record({
+            :stream_name   => @stream_name,
+            :data          => "["+bufList[":#{data['pk']}"].chop+"]",
+            :partition_key => data['pk']
+          })
+          bufList.delete(":#{data['pk']}")
+        end
+      end
+      dataList.each do |data|
+        if bufList[":#{data['pk']}"] != nil then
+          AWS.kinesis.client.put_record({
+            :stream_name   => @stream_name,
+            :data          => "["+bufList[":#{data['pk']}"].chop+"]",
+            :partition_key => data['pk']
+          })
+          bufList.delete(":#{data['pk']}")
+        end
+      end
+    end
+
+    private
+
+    def configure_aws
+      options = {
+        :access_key_id     => @access_key_id,
+        :secret_access_key => @secret_access_key,
+        :region            => @region
+      }
+
+      if @debug
+        options.update(
+          :logger          => Logger.new($log.out),
+          :log_level       => :debug,
+          #http_wire_trace => true
+        )
+      end
+
+      AWS.config(options)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+
+  nulllogger.instance_eval do |obj|
+    def method_missing(method, *args); end
+  end
+
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_kinesis_alt'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_grassland.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+class KinesisAltOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+  ]
+  # CONFIG = %[
+  #   path #{TMP_DIR}/out_file_test
+  #   compress gz
+  #   utc
+  # ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::KinesisAltOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    #### set configurations
+    # d = create_driver %[
+    #   path test_path
+    #   compress gz
+    # ]
+    #### check configurations
+    # assert_equal 'test_path', d.instance.path
+    # assert_equal :gz, d.instance.compress
+  end
+
+  def test_format
+    #d = create_driver
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n]
+    # d.expect_format %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    # d.run
+  end
+
+  def test_write
+    #d = create_driver
+
+    # time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    # d.emit({"a"=>1}, time)
+    # d.emit({"a"=>2}, time)
+
+    # ### FileOutput#write returns path
+    # path = d.run
+    # expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
+    # assert_equal expect_path, path
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-grassland
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ripplation Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.38'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.38'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Output filter plugin for Grassland
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-grassland.gemspec
+- lib/fluent/plugin/out_grassland.rb
+- test/helper.rb
+- test/plugin/test_out_grassland.rb
+homepage: https://bitbucket.org/winebarrel/fluent-plugin-kinesis-alt
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Output filter plugin for Grassland
+test_files:
+- test/helper.rb
+- test/plugin/test_out_grassland.rb