fluent-plugin-td 0.9.7

data/AUTHORS

@@ -0,0 +1 @@
+FURUHASHI Sadayuki <frsyuki _at_ gmail.com>

data/README.rdoc

@@ -0,0 +1,7 @@
+= Treasure Data output plugin for Fluent
+
+== Copyright
+
+Copyright:: Copyright (c) 2011 Treasure Data, Inc.
+License::   Apache License, Version 2.0
+

data/Rakefile

@@ -0,0 +1,50 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "fluent-plugin-td"
+    gemspec.summary = "Treasure Data plugin for Fluent event collector"
+    gemspec.author = "Sadayuki Furuhashi"
+    #gemspec.email = "frsyuki@gmail.com"
+    #gemspec.homepage = "http://fluent.github.com/"
+    gemspec.has_rdoc = false
+    gemspec.require_paths = ["lib"]
+    gemspec.add_dependency "fluent", "~> 0.9.7"
+    gemspec.test_files = Dir["test/**/*.rb"]
+    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
+      %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-td.gemspec]
+    gemspec.executables = []
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/*_test.rb']
+  t.ruby_opts = ['-rubygems'] if defined? Gem
+  t.ruby_opts << '-I.'
+end
+
+#VERSION_FILE = "lib/fluent/version.rb"
+#
+#file VERSION_FILE => ["VERSION"] do |t|
+#  version = File.read("VERSION").strip
+#  File.open(VERSION_FILE, "w") {|f|
+#    f.write <<EOF
+#module Fluent
+#
+#VERSION = '#{version}'
+#
+#end
+#EOF
+#  }
+#end
+#
+#task :default => [VERSION_FILE, :build]
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.9.7

data/example.conf

@@ -0,0 +1,78 @@
+
+## match tag=apache.access and upload to Treasure Data
+#<match apache.access>
+#  type tdlog
+#  apikey APIKEY
+#</match>
+
+
+## built-in TCP input
+<source>
+  type tcp
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  type http
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  type file
+#  path /var/log/td-agent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+## match tag=system.** and forward to another td-agent server
+#<match system.**>
+#  type tcp
+#  host 192.168.0.11
+#  <secondary>
+#    host 192.168.0.12
+#  </secondary>
+#</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  type copy
+#  <store>
+#    type tcp
+#    host 192.168.0.13
+#    buffer_type file
+#    buffer_path /var/log/td-agent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#  </store>
+#  <store>
+#    type file
+#    path /var/log/td-agent/myapp
+#  </store>
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  type file
+#  path /var/log/td-agent/else/%Y-%m-%d/%H.log
+#</match>
+

data/fluent-plugin-td.gemspec

@@ -0,0 +1,42 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{fluent-plugin-td}
+  s.version = "0.9.7"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Sadayuki Furuhashi"]
+  s.date = %q{2011-08-06}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    "AUTHORS",
+     "Rakefile",
+     "VERSION",
+     "example.conf",
+     "fluent-plugin-td.gemspec",
+     "lib/fluent/plugin/out_tdlog.rb"
+  ]
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Treasure Data plugin for Fluent event collector}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fluent>, ["~> 0.9.7"])
+    else
+      s.add_dependency(%q<fluent>, ["~> 0.9.7"])
+    end
+  else
+    s.add_dependency(%q<fluent>, ["~> 0.9.7"])
+  end
+end
+

data/lib/fluent/plugin/out_tdlog.rb

@@ -0,0 +1,209 @@
+module Fluent
+
+
+class TreasureDataLogOutput < BufferedOutput
+  Plugin.register_output('tdlog', self)
+
+  HOST = ENV['TD_API_SERVER'] || 'api.treasure-data.com'
+  PORT = 80
+  USE_SSL = false
+
+  def initialize
+    require 'fileutils'
+    require 'tempfile'
+    require 'zlib'
+    require 'net/http'
+    require 'json'
+    require 'cgi' # CGI.escape
+    require 'time' # Time#rfc2822
+    super
+    @tmpdir = '/tmp/fluent/tdlog'
+    @apikey = nil
+    @key = nil
+    @table_list = []
+  end
+
+  def configure(conf)
+    super
+
+    @tmpdir = conf['tmpdir'] || @tmpdir
+    FileUtils.mkdir_p(@tmpdir)
+
+    @apikey = conf['apikey']
+    unless @apikey
+      raise ConfigError, "'apikey' parameter is required on tdlog output"
+    end
+
+    database = conf['database']
+    table = conf['table']
+    if database && table
+      if !validate_name(database)
+        raise ConfigError, "Invalid database name #{database.inspect}: #{conf}"
+      end
+      if !validate_name(table)
+        raise ConfigError, "Invalid table name #{table.inspect}: #{conf}"
+      end
+      @key = "#{database}.#{table}"
+    elsif (database && !table) || (!database && table)
+      raise ConfigError, "'database' and 'table' parameter are required on tdlog output"
+    end
+
+    @table_list = get_table_list
+
+    if @key && !@table_list.include?(@key)
+      raise ConfigError, "Table #{@key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
+    end
+  end
+
+  def emit(tag, es, chain)
+    if @key
+      key = @key
+    else
+      database, table = tag.split('.')[-2,2]
+      if !validate_name(database) || !validate_name(table)
+        $log.debug { "Invalid tag #{tag.inspect}" }
+        return
+      end
+      key = "#{database}.#{table}"
+    end
+
+    # check the table exists
+    unless @table_list.include?(key)
+      begin
+        @table_list = get_table_list
+      rescue
+        $log.warn "failed to update table list on Treasure Data", :error=>$!.to_s
+        $log.debug_backtrace $!
+      end
+      unless @table_list.include?(key)
+        database, table = key.split('.',2)
+        raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td create-log-table #{database} #{table}' to create it."
+      end
+    end
+
+    super(tag, es, chain, key)
+  end
+
+  def validate_name(name)
+    true
+  end
+
+  def format_stream(tag, es)
+    out = ''
+    es.each {|event|
+      record = event.record
+      record['time'] = event.time
+      record.to_msgpack(out)
+    }
+    out
+  end
+
+  def write(chunk)
+    database, table = chunk.key.split('.',2)
+    if !validate_name(database) || !validate_name(table)
+      $log.error "Invalid key name #{chunk.key.inspect}"
+      return
+    end
+
+    f = Tempfile.new("tdlog-", @tmpdir)
+    w = Zlib::GzipWriter.new(f)
+
+    chunk.write_to(w)
+    w.finish
+    w = nil
+
+    size = f.pos
+    f.pos = 0
+    upload(database, table, f, size)
+
+  ensure
+    w.close if w
+    f.close if f
+  end
+
+  def upload(database, table, io, size)
+    http, header = new_http
+    header['Content-Length'] = size.to_s
+    header['Content-Type'] = 'application/octet-stream'
+
+    url = "/v3/table/import/#{e database}/#{e table}/msgpack.gz"
+
+    req = Net::HTTP::Put.new(url, header)
+    if req.respond_to?(:body_stream=)
+      req.body_stream = io
+    else  # Ruby 1.8
+      req.body = io.read
+    end
+
+    $log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
+
+    response = http.request(req)
+
+    if response.code[0] != ?2
+      raise "Treasure Data upload failed: #{response.body}"
+    end
+  end
+
+  def get_table_list
+    $log.info "updating table list from Treasure Data"
+    list = []
+    api_list_database.each {|db|
+      api_list_table(db).each {|t|
+        list << "#{db}.#{t}"
+      }
+    }
+    list
+  end
+
+  def api_list_database
+    body = get("/v3/database/list")
+    js = JSON.load(body)
+    return js["databases"].map {|m| m['name'] }
+  end
+
+  def api_list_table(db)
+    body = get("/v3/table/list/#{e db}")
+    js = JSON.load(body)
+    return js["tables"].map {|m| m['name'] }
+  end
+
+  def get(path)
+    http, header = new_http
+
+    request = Net::HTTP::Get.new(path, header)
+
+    response = http.request(request)
+
+    if response.code[0] != ?2
+      raise "Treasure Data API failed: #{response.body}"
+    end
+
+    return response.body
+  end
+
+  def new_http
+    http = Net::HTTP.new(HOST, PORT)
+    if USE_SSL
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      store = OpenSSL::X509::Store.new
+      http.cert_store = store
+    end
+
+    # TODO read_timeout
+    #http.read_timeout = options[:read_timeout]
+
+    header = {}
+    header['Authorization'] = "TD1 #{@apikey}"
+    header['Date'] = Time.now.rfc2822
+
+    return http, header
+  end
+
+  def e(s)
+    CGI.escape(s.to_s)
+  end
+end
+
+
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: fluent-plugin-td
+version: !ruby/object:Gem::Version 
+  hash: 53
+  prerelease: false
+  segments: 
+  - 0
+  - 9
+  - 7
+  version: 0.9.7
+platform: ruby
+authors: 
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-06 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fluent
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 53
+        segments: 
+        - 0
+        - 9
+        - 7
+        version: 0.9.7
+  type: :runtime
+  version_requirements: *id001
+description: 
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- AUTHORS
+- Rakefile
+- VERSION
+- example.conf
+- fluent-plugin-td.gemspec
+- lib/fluent/plugin/out_tdlog.rb
+- README.rdoc
+has_rdoc: true
+homepage: 
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Treasure Data plugin for Fluent event collector
+test_files: []
+