fluent-plugin-swift-sweep 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7eb98fc312888473404f3e39c21c40e503d92858
+  data.tar.gz: a8227b875c7f008ea866b35f03df7cac3af53940
+SHA512:
+  metadata.gz: 5babdc86027cd29d1896baeea3a2f67d0684291796d55a93155924238f13fc700f61c63adc00a2bafee22f8d2ecda0ee3085eb01fa6b7fd09f06404827fbd199
+  data.tar.gz: afd48f1cfb3cbf0934adfe9b3d9720148c8c97ca0eb96dbcd1d28c5a3aced4554637bc93661fbfb70bc1c2f27f14cbf4613a2faa2a730792c8cc43eedc77ac32

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 0.0.1
+
+Enhancements:
+
+* Initial (debug log)
+
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-swift-sweep.gemspec 
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 deepak.arumugham 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# fluent-plugin-swift-sweep
+
+
+Fluentd plugin to read data from files and move it Swift object storage as is.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-swift-sweep'
+```
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-swift-sweep
+
+## Basic Behavior
+
+Assume your files are inside `/tmp/test` directory as
+
+```
+tmp/test
+├── core.files1.log
+├── core.files2.log
+└── core.files2.log
+```
+
+
+
+This plugin watches the directory (`file_path_with_glob tmp/test/*.log`), and reads the contents and sends the files to swift storage and removes the file, after sending them to swift.
+
+The files will be moved to swift container as:
+tmp/test/core.files1.log
+tmp/test/core.files2.log
+tmp/test/core.files3.log
+
+## Configuration
+
+```
+<source>
+  type swift_swwep
+
+  # Required. process files that match this pattern using glob.
+  file_path_with_glob /tmp/imp/*.log
+
+  # Required. Authentication URL 
+  auth_url <Authentication url>
+
+  # Required. Authenticated User Name
+  auth_user <User name>
+
+  # Required. Password
+  auth_api_key <Password>
+
+  # Required. The name of the openstack tenant
+  auth_tenant <Openstack Tenant>
+
+  # Required. The name of the swift container
+  swift_container <Container Name>
+
+  ssl_verify false
+</source>
+```
+
+## ChangeLog
+
+[CHANGELOG.md](CHANGELOG.md)
+
+## Warning
+* This plugin supports fluentd from v0.12.33
+
+## Contributing
+
+1. Fork it ( https://github.com/deepakarumugham/fluent-plugin-swift-sweep/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/fluent-plugin-swift-sweep.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-swift-sweep"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Deepak Arumugham"]
+  spec.email         = ["deepak.arumugham@gmail.com"]
+
+  spec.summary       = %q{Fluentd plugin to move files to swift container}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/deepakarumugham/fluent-plugin-swift-sweep"
+  spec.license       = "MIT"
+
+  spec.files         = `find .`.split("\n")
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "fluentd", "~> 0.12.0"
+  spec.add_dependency "fog", "~> 1.15.0"
+end

data/gemfiles/Gemfile.fluentd.0.12

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+
+gem 'fluentd', '~> 0.12.0'
+gemspec :path => "../"

data/lib/fluent/plugin/in_swift_sweep.rb

@@ -0,0 +1,105 @@
+module Fluent
+  class SwiftSweepInput < Input
+    Plugin.register_input('swift_sweep', self)
+    
+    def initialize
+      super
+      require 'fog'
+      require 'time'
+      require 'open3'
+    end
+
+    config_param :file_path_with_glob,     :string
+    config_param :auth_url, :string
+    config_param :auth_user, :string
+    config_param :auth_tenant, :string
+    config_param :auth_api_key, :string
+    config_param :swift_container, :string
+    config_param :auto_create_container, :bool, :default => true
+    config_param :ssl_verify, :bool, :default => false
+
+
+    def configure(conf)
+      super
+
+      if @file_path_with_glob.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@file_path_with_glob` must has a valid path."
+      end
+
+      if @auth_url.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@auth_url` is empty."
+      end
+
+      if @auth_user.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@auth_user` is empty."
+      end
+
+      if @auth_tenant.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@auth_tenant` is empty."
+      end
+
+      if @auth_api_key.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@auth_api_key` is empty."
+      end
+
+      if @swift_container.empty?
+        raise Fluent::ConfigError, "in_swift_sweep: `@swift_container` is empty."
+      end
+
+    end
+
+    def start
+      super
+
+      Excon.defaults[:ssl_verify_peer] = false
+
+      $log.debug "openstack_auth_url: #{@auth_url}"
+      $log.debug "openstack_username: #{@auth_user}"
+      $log.debug "openstack_tenant: #{@auth_tenant}"
+      $log.debug "openstack_auth_key: #{@auth_api_key}"
+      @storage = Fog::Storage.new :provider => 'OpenStack',
+                        :openstack_auth_url => @auth_url,
+                        :openstack_username => @auth_user,
+                        :openstack_tenant => @auth_tenant,
+                        :openstack_api_key  => @auth_api_key
+
+      check_container
+      
+      @processing = true
+      @thread = Thread.new(&method(:run_periodic))
+    end
+
+    def shutdown
+      @processing = false
+      @thread.join
+    end
+
+    def run_periodic
+      while @processing
+        Dir.glob(@file_path_with_glob).map do |filename|
+          File.open(filename) do |file|
+            @storage.put_object(@swift_container, filename, file)
+            log.info "File #{filename} sent to swift"
+            FileUtils.rm(filename)
+            log.info "File #{filename} deleted"
+          end
+        end
+      end
+    end
+
+
+    def check_container
+      begin
+        @storage.get_container(@swift_container)
+      rescue Fog::Storage::OpenStack::NotFound
+        if @auto_create_container
+          $log.info "Creating container #{@swift_container} on #{@auth_url}, #{@swift_account}"
+          @storage.put_container(@swift_container)
+        else
+          raise "The specified container does not exist: container = #{swift_container}"
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-swift-sweep
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Deepak Arumugham
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.0
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+description: Fluentd plugin to move files to swift container
+email:
+- deepak.arumugham@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- "./CHANGELOG.md"
+- "./Gemfile"
+- "./LICENSE.txt"
+- "./README.md"
+- "./fluent-plugin-swift-sweep.gemspec"
+- "./gemfiles/Gemfile.fluentd.0.12"
+- "./lib/fluent/plugin/in_swift_sweep.rb"
+homepage: https://github.com/deepakarumugham/fluent-plugin-swift-sweep
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to move files to swift container
+test_files: []