fluent-plugin-oss 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 479e3eae96e9f6f94d41e0f9a4bd8ba126517f1b
+  data.tar.gz: 4cf891a7f1917911a4b280ab1d46dc1739519aa6
+SHA512:
+  metadata.gz: b059f115c4236d68ef55d8105fd1044e915d62be5f8707aa16199a27f6d9cb1ca66b6f74966c255757f669b2f4d0aabfe5d8ac0cc35be2009217dbfb5be42474
+  data.tar.gz: d98ff286e683e6b5acc75a2e81b295fb318559c995f5699a03f30ccfdcd3d503a721185a65c736aa4f02e0a7257139844fb6657008f7d524a94433405e8baa52

data/.gitignore

@@ -0,0 +1,13 @@
+~*
+#*
+*~
+[._]*.s[a-w][a-z]
+.DS_Store
+
+*.gem
+.bundle
+Gemfile.lock
+vendor
+.ruby-version
+
+test/tmp/

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,19 @@
+# OSS plugin for [Fluentd](http://github.com/fluent/fluentd)
+
+## Output: Configuration
+
+```
+<match **>
+  @type oss
+  oss_key_id xxx
+  oss_key_secret xxx
+  oss_bucket xxx
+  oss_endpoint xxx
+  oss_object_key_format "%{time_slice}/%{host}-%{uuid}.%{file_ext}"
+
+  buffer_path /var/log/fluent/myapp
+  time_slice_format %Y%m%d
+  time_slice_wait 10m
+  time_format %Y%m%dT%H%M%S%z
+</match>
+```

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/fluent-plugin-oss.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-oss"
+  gem.description = "Aliyun oss output plugin for Fluentd event collector"
+  gem.license     = "MIT"
+  gem.homepage    = "https://github.com/jicong/fluent-plugin-oss"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["sjtubreeze"]
+  gem.email       = "sjtubreeze@163.com"
+  gem.has_rdoc    = false
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.12.3"]
+  gem.add_dependency "aliyun-sdk", [">= 0.6.0"]
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
+end

data/lib/fluent/plugin/out_oss.rb

@@ -0,0 +1,85 @@
+require 'fluent/output'
+require 'fluent/log'
+require 'aliyun/oss'
+require 'zlib'
+require 'time'
+require 'tempfile'
+require 'securerandom'
+require 'socket'
+
+module Fluent
+  class OSSOutput < TimeSlicedOutput
+    Fluent::Plugin.register_output('oss', self)
+
+    desc "OSS access key id"
+    config_param :oss_key_id, :string
+    desc "OSS access key secret"
+    config_param :oss_key_secret, :string, secret: true
+    desc "OSS bucket name"
+    config_param :oss_bucket, :string
+    desc "OSS endpoint"
+    config_param :oss_endpoint, :string
+    desc "The format of OSS object keys"
+    config_param :oss_object_key_format, :string, default: "%{time_slice}/%{host}-%{uuid}.%{file_ext}"
+
+    def configure(conf)
+      super
+    end
+
+    def compress(chunk, tmp)
+      res = system "gzip -c #{chunk.path} > #{tmp.path}"
+      unless res
+        log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+        begin
+          tmp.truncate(0)
+          gw = Zlib::GzipWriter.new(tmp)
+          chunk.write_to(gw)
+          gw.close
+        ensure
+          gw.close rescue nil
+        end
+      end
+    end
+
+    def process_object_key_format(chunk, key_format)
+      key_map = {
+        host: Socket.gethostname,
+        time_slice: chunk.key,
+        uuid: SecureRandom.hex(4),
+        file_ext: 'gz'
+      }
+      result = key_format
+      key_map.each do |k, v|
+        result = result.gsub("%{#{k.to_s}}", v)
+      end
+      result
+    end
+
+    def start
+      super
+      @client = Aliyun::OSS::Client.new(
+        :endpoint => @oss_endpoint,
+        :access_key_id => @oss_key_id,
+        :access_key_secret => @oss_key_secret)
+
+      raise "Specific bucket not exists: #{@oss_bucket}" unless @client.bucket_exists? @oss_bucket
+
+      @bucket = @client.get_bucket(@oss_bucket)
+    end
+
+    def format(tag, time, record)
+      {tag: tag, timestamp: time, log: record}.to_json + "\n"
+    end
+
+    def write(chunk)
+      begin
+        f = Tempfile.new('oss-')
+        compress(chunk, f)
+        path = process_object_key_format(chunk, @oss_object_key_format)
+        raise "Upload #{f.path} failed" unless @bucket.resumable_upload(path, f.path)
+      ensure
+        f.close(true)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-oss
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- sjtubreeze
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.12.3
+- !ruby/object:Gem::Dependency
+  name: aliyun-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+description: Aliyun oss output plugin for Fluentd event collector
+email: sjtubreeze@163.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- VERSION
+- fluent-plugin-oss.gemspec
+- lib/fluent/plugin/out_oss.rb
+homepage: https://github.com/jicong/fluent-plugin-oss
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Aliyun oss output plugin for Fluentd event collector
+test_files: []