fluent-plugin-ftp 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bae24cf10e0a1de9456803e8cfded0a8b8ff1ad6
+  data.tar.gz: e48fbb7ffbe75d03d69f76d5eb933a631f87ca9a
+SHA512:
+  metadata.gz: 08c0a72cfa182122189e1bc0216659894dbbacb3edc44f59fdc7e53b29d1582f9a032a5a11d1c6f0430c25bf19484e2a3add3677df2ed069496dbebc805ed095
+  data.tar.gz: 7df1f757450443171330ce98d8f95f997bbf050e814cc38c93da4f604fa5679a2b2c467425fcda3955a43875553b05657fb506d32f80a972f7c8f7c13b10991a

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - rbx-19mode
+
+branches:
+  only:
+    - master
+
+script: bundle exec rake test

data/AUTHORS

@@ -0,0 +1 @@
+Kazuki Ohta <kazuki.ohta _at_ gmail.com>

data/ChangeLog

@@ -0,0 +1,4 @@
+Release 0.1.1 - 2011/09/27
+
+  * First release
+

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,46 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-ftp (0.0.1)
+      aws-sdk (~> 1.8.2)
+      fluent-mixin-config-placeholders (~> 0.2.0)
+      fluentd (~> 0.10.0)
+      yajl-ruby (~> 1.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    aws-sdk (1.8.5)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+      uuidtools (~> 2.1)
+    cool.io (1.2.1)
+    flexmock (1.3.3)
+    fluent-mixin-config-placeholders (0.2.4)
+      fluentd
+      uuidtools
+    fluentd (0.10.44)
+      cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
+      http_parser.rb (~> 0.5.1)
+      json (>= 1.4.3)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      sigdump (~> 0.2.2)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.5.3)
+    json (1.8.1)
+    mini_portile (0.5.2)
+    msgpack (0.5.8)
+    nokogiri (1.6.1)
+      mini_portile (~> 0.5.0)
+    rake (10.1.1)
+    sigdump (0.2.2)
+    uuidtools (2.1.4)
+    yajl-ruby (1.2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  flexmock (>= 1.2.0)
+  fluent-plugin-ftp!
+  rake (>= 0.9.2)

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/example.conf

@@ -0,0 +1,24 @@
+<source>
+  type forward
+</source>
+
+<match debug.**>
+  type copy
+  <store>
+    type stdout
+  </store>
+  <store>
+    type ftp
+    ftp_server   localhost
+    ftp_user     user
+    ftp_password pass
+    ftp_dir      dir
+    
+    buffer_path buffer_dir
+    flush_interval 3s
+
+    time_slice_format %Y%m%d%H
+    time_slice_wait 10m
+    utc
+  </store>
+</match>

data/fluent-plugin-ftp.gemspec

@@ -0,0 +1,25 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-ftp"
+  gem.description = "FTP input / output plugin for Fluentd data collector"
+  gem.homepage    = "https://github.com/kzk/fluent-plugin-ftp"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kazuki Ohta"]
+  gem.email       = "kazuki.ohta@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "aws-sdk", "~> 1.8.2"
+  gem.add_dependency "yajl-ruby", "~> 1.0"
+  gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "flexmock", ">= 1.2.0"
+end

data/lib/fluent/plugin/out_ftp.rb

@@ -0,0 +1,174 @@
+module Fluent
+  require 'fluent/mixin/config_placeholders'
+
+  class FTPOutput < Fluent::TimeSlicedOutput
+    Fluent::Plugin.register_output('ftp', self)
+
+    def initialize
+      super
+      require 'net/ftp'
+      require 'zlib'
+      require 'time'
+      require 'tempfile'
+      require 'open3'
+      require 'pathname'
+    end
+
+    config_param :path, :string, :default => ""
+    config_param :time_format, :string, :default => nil
+
+    include SetTagKeyMixin
+    config_set_default :include_tag_key, false
+
+    include SetTimeKeyMixin
+    config_set_default :include_time_key, false
+
+    config_param :ftp_server, :string
+    config_param :ftp_user, :string
+    config_param :ftp_password, :string
+    config_param :ftp_dir, :string
+    config_param :ftp_file_name_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
+    config_param :store_as, :string, :default => "gzip"
+
+    include Fluent::Mixin::ConfigPlaceholders
+
+    def placeholders
+      [:percent]
+    end
+
+    def configure(conf)
+      super
+
+      if format_json = conf['format_json']
+        @format_json = true
+      else
+        @format_json = false
+      end
+
+      @timef = TimeFormatter.new(@time_format, @localtime)
+
+      @ext = case @store_as
+        when 'gzip' then 'gz'
+        when 'lzo' then
+          begin
+            Open3.capture3('lzop -V')
+          rescue Errno::ENOENT
+            raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
+          end
+          'lzo'
+        when 'json' then 'json'
+        else 'txt'
+      end
+
+      if @localtime
+        @path_slicer = Proc.new { |path| Time.now.strftime(path) }
+      else
+        @path_slicer = Proc.new { |path| Time.now.utc.strftime(path) }
+      end
+    end
+
+    def start
+      super
+      with_ftp_connection { |conn|
+        # check valid connection here, nothing to do
+      }
+    end
+
+    def format(tag, time, record)
+      if @include_time_key || !@format_json
+        time_str = @timef.format(time)
+      end
+
+      # copied from each mixin because current TimeSlicedOutput can't support mixins.
+      if @include_tag_key
+        record[@tag_key] = tag
+      end
+      if @include_time_key
+        record[@time_key] = time_str
+      end
+
+      if @format_json
+        Yajl.dump(record) + "\n"
+      else
+        "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
+      end
+    end
+
+    def write(chunk)
+      with_ftp_connection { |conn|
+        ftp_filename = gen_ftp_filename(conn, chunk.key)
+        with_compression(@store_as, chunk) { |tmp_path|
+          conn.putbinaryfile(tmp_path, ftp_filename)
+        }
+      }
+    end
+
+    private
+
+    def with_ftp_connection
+      ftp = Net::FTP.new
+      ftp.binary = true
+      begin
+        ftp.connect(@ftp_server)
+        ftp.login(@ftp_user, @ftp_password)
+        ftp.chdir(@ftp_dir)
+        yield ftp
+      ensure
+        ftp.quit() rescue nil
+      end
+    end
+
+    def gen_ftp_filename(conn, chunk_key)
+      i = 0
+      begin
+        values_for_ftp_file_name = {
+          "path" => @path_slicer.call(@path),
+          "time_slice" => chunk_key,
+          "file_extension" => @ext,
+          "index" => i
+        }
+        ftp_file_name = @ftp_file_name_format.gsub(%r(%{[^}]+})) { |expr|
+          values_for_ftp_file_name[expr[2...expr.size-1]]
+        }
+        i += 1
+      end while ftp_file_exists?(conn, ftp_file_name)
+      return ftp_file_name
+    end
+
+    def ftp_file_exists?(conn, filename)
+      begin
+        not conn.nlst(filename).empty?
+      rescue Net::FTPTempError => e
+        err_code = e.message[0,3].to_i
+        return false if err_code == 450
+        raise e
+      end
+    end
+
+    def with_compression(codec, chunk)
+      tmp = Tempfile.new("ftp-")
+      begin
+        if codec == "gzip"
+          w = Zlib::GzipWriter.new(tmp)
+          chunk.write_to(w)
+          w.close
+        elsif codec == "lzo"
+          w = Tempfile.new("chunk-tmp")
+          chunk.write_to(w)
+          w.close
+          tmp.close
+          # we don't check the return code because we can't recover lzop failure.
+          system "lzop -qf1 -o #{tmp.path} #{w.path}"
+        else
+          chunk.write_to(tmp)
+          tmp.close
+        end
+        yield Pathname.new(tmp.path)
+      ensure
+        tmp.close(true) rescue nil
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-ftp
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kazuki Ohta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-03-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: FTP input / output plugin for Fluentd data collector
+email: kazuki.ohta@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".travis.yml"
+- AUTHORS
+- ChangeLog
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- VERSION
+- example.conf
+- fluent-plugin-ftp.gemspec
+- lib/fluent/plugin/out_ftp.rb
+homepage: https://github.com/kzk/fluent-plugin-ftp
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: FTP input / output plugin for Fluentd data collector
+test_files: []