fluent-plugin-td 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8e132d9df4dd5ed1344497a0680b12caa75c8d9b6361feef6d36e6d50eef0d2
-  data.tar.gz: 475920dcb5e14528ce9e2c5d5baab44811fa1fcab9b613f22c8212e10e56cf4c
+  metadata.gz: a08410b0e8078a20d2150bacc0cde7cc14cfb70be600fbcc86309b5df1b7076c
+  data.tar.gz: 4aca6ca191fa4401f198b323cd0ae12c1616de67e640bc8111237af6eca8f976
 SHA512:
-  metadata.gz: bf337f321840654a6ace12117b899234bfe9b144756d96f34986381e84eb8a9f4d6d2f970ae9d55a3b49ed4fce6bf3c9d514aaaaa6e44c441f567c75d3238b0a
-  data.tar.gz: 74fc971d23dac8608901ff7bcc9add4c969f20bdf3a293a653f26ebc703e3b951479ac9e98d296f83e222ad3dc8fc0c6636e5a0f1533e4e2fa6397225b7cc79b
+  metadata.gz: a776e4f86f5793451a04e76a9db1eb746ffb762d53e3781f1f20c8187663dcd8c575ef9814207c546831bd6192674661c6c38041998928f98ec159bbcd40d85c
+  data.tar.gz: 03a5a89b24a49813255dbe340b84b9dcd3abba74373dbb655d2220f3803f6beac8dbe6e9211312750ef00f59cf0f0b7e794c68810321b00aab20d4124d1997e3

data/.circleci/config.yml

@@ -0,0 +1,51 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@1.8
+jobs:
+  test:
+    parameters:
+      ruby-version:
+        type: string
+    docker:
+      - image: cimg/ruby:<< parameters.ruby-version >>
+    steps:
+      - checkout
+      - ruby/install-deps
+      - run:
+          name: Run tests
+          command: bundle exec rake test
+  publish:
+    docker:
+      - image: cimg/ruby:2.4
+    steps:
+      - checkout
+      - ruby/install-deps
+      - run:
+          name: Publish gem
+          command: |
+            mkdir -p $HOME/.gem
+            touch $HOME/.gem/credentials
+            chmod 0600 $HOME/.gem/credentials
+            printf -- "---\n:rubygems_api_key: ${RUBYGEMS_AUTH_TOKEN}\n" > $HOME/.gem/credentials
+            gem build *.gemspec
+            gem push *.gem
+workflows:
+  version: 2
+  test:
+    jobs:
+      - test:
+          matrix:
+            parameters:
+              ruby-version: [ "2.4", "2.5", "2.6", "2.7", "3.0", "3.1" ]
+          filters:
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/
+      - publish:
+          context: rubygems
+          requires:
+            - test
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /v[0-9]+(\.[0-9]+)*/

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,31 @@
+name: "CodeQL"
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '37 9 * * 3'
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'ruby' ]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

data/.gitignore

@@ -1,2 +1,2 @@
-Gemfile.lock
 pkg/*
+.idea

data/Gemfile

@@ -1,3 +1,3 @@
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gemspec

data/Gemfile.lock

@@ -0,0 +1,95 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-td (1.2.0)
+      fluentd (>= 0.14.13, < 2)
+      td-client (>= 1.0.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    concurrent-ruby (1.1.10)
+    cool.io (1.7.1)
+    crack (0.4.5)
+      rexml
+    fluentd (1.14.6)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.9.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.5, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.4, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    hashdiff (1.0.1)
+    http_parser.rb (0.8.0)
+    httpclient (2.8.3)
+    msgpack (1.5.1)
+    parallel (1.20.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    power_assert (2.0.1)
+    public_suffix (4.0.7)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.4.0)
+    rexml (3.2.5)
+    rr (3.0.9)
+    rubocop (1.12.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    rubocop-rake (0.5.1)
+      rubocop
+    ruby-progressbar (1.11.0)
+    serverengine (2.2.5)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.5)
+    td-client (1.0.8)
+      httpclient (>= 2.7)
+      msgpack (>= 0.5.6, < 2)
+    test-unit (3.5.3)
+      power_assert
+    test-unit-rr (1.0.5)
+      rr (>= 1.1.1)
+      test-unit (>= 2.5.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2022.1)
+      tzinfo (>= 1.0.0)
+    unicode-display_width (2.1.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-td!
+  rake (>= 0.9.2)
+  rubocop
+  rubocop-rake
+  test-unit (~> 3.5.3)
+  test-unit-rr (~> 1.0.5)
+  webmock (~> 3.14.0)
+
+BUNDLED WITH
+   2.1.4

data/fluent-plugin-td.gemspec

@@ -1,26 +1,26 @@
-# encoding: utf-8
-$:.push File.expand_path('../lib', __FILE__)
-require 'fluent/plugin/td_plugin_version'
+require File.expand_path('../lib/fluent/plugin/td_plugin_version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.name        = "fluent-plugin-td"
-  gem.description = "Treasure Data Cloud Data Service plugin for Fluentd"
-  gem.homepage    = "http://www.treasuredata.com/"
+  gem.name        = 'fluent-plugin-td'
+  gem.description = 'Treasure Data Cloud Data Service plugin for Fluentd'
+  gem.homepage    = 'https://www.treasuredata.com/'
   gem.summary     = gem.description
   gem.version     = Fluent::Plugin::TreasureDataPlugin::VERSION
-  gem.authors     = ["Treasure Data, Inc."]
-  gem.email       = "support@treasure-data.com"
-  #gem.platform    = Gem::Platform::RUBY
+  gem.authors     = ['Treasure Data, Inc.']
+  gem.email       = 'support@treasure-data.com'
   gem.files       = `git ls-files`.split("\n")
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   gem.require_paths = ['lib']
-  gem.license       = "Apache-2.0"
+  gem.license       = 'Apache-2.0'
 
-  gem.add_dependency "fluentd", [">= 0.14.13", "< 2"]
-  gem.add_dependency "td-client", "~> 1.0"
-  gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency "webmock", "~> 1.16"
-  gem.add_development_dependency "test-unit", "~> 3.0.8"
-  gem.add_development_dependency "test-unit-rr", "~> 1.0.3"
+  gem.required_ruby_version = '>= 2.4'
+  gem.add_dependency 'fluentd', ['>= 0.14.13', '< 2']
+  gem.add_dependency 'td-client', '>= 1.0.8'
+  gem.add_development_dependency 'rake', '>= 0.9.2'
+  gem.add_development_dependency 'rubocop'
+  gem.add_development_dependency 'rubocop-rake'
+  gem.add_development_dependency 'test-unit', '~> 3.5.3'
+  gem.add_development_dependency 'test-unit-rr', '~> 1.0.5'
+  gem.add_development_dependency 'webmock', '~> 3.14.0'
 end

data/lib/fluent/plugin/out_tdlog.rb

@@ -4,6 +4,7 @@ require 'zlib'
 require 'stringio'
 require 'td-client'
 
+require 'fluent/error'
 require 'fluent/plugin/output'
 require 'fluent/plugin/td_plugin_version'
 
@@ -12,6 +13,7 @@ module Fluent::Plugin
     Fluent::Plugin.register_output('tdlog', self)
 
     IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
+    IMPORT_RECORDS_LIMIT = 8096
     UPLOAD_EXT = 'msgpack.gz'.freeze
 
     helpers :event_emitter, :compat_parameters
@@ -36,13 +38,14 @@ module Fluent::Plugin
       config_set_default :chunk_keys, ['tag']
       config_set_default :flush_interval, 300
       config_set_default :chunk_limit_size, IMPORT_SIZE_LIMIT
+      config_set_default :chunk_limit_records, IMPORT_RECORDS_LIMIT
     end
 
     def initialize
       super
       @key = nil
-      @key_num_limit = 512  # TODO: Our one-time import has the restriction about the number of record keys.
-      @record_size_limit = 32 * 1024 * 1024  # TODO
+      @key_num_limit = 512
+      @record_size_limit = 32 * 1024 * 1024
       @table_list = {}
       @empty_gz_data = TreasureData::API.create_empty_gz_data
       @user_agent = "fluent-plugin-td: #{TreasureDataPlugin::VERSION}".freeze
@@ -107,7 +110,6 @@ module Fluent::Plugin
         record.delete(:time) if record.has_key?(:time)
 
         if record.size > @key_num_limit
-          # TODO include summary of the record
           router.emit_error_event(tag, time, record, RuntimeError.new("too many number of keys (#{record.size} keys)"))
           return nil
         end
@@ -168,7 +170,6 @@ module Fluent::Plugin
       f.close(true) if f
     end
 
-    # TODO: Share this routine with s3 compressors
     def gzip_by_command(chunk, tmp)
       chunk_is_file = @buffer_config['@type'] == 'file'
       path = if chunk_is_file
@@ -183,10 +184,8 @@ module Fluent::Plugin
       res = system "gzip -c #{path} > #{tmp.path}"
       unless res
         log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
-        begin
-          tmp.truncate(0)
-          return gzip_by_writer(chunk, tmp)
-        end
+        tmp.truncate(0)
+        return gzip_by_writer(chunk, tmp)
       end
       File.size(tmp.path)
     ensure
@@ -211,18 +210,22 @@ module Fluent::Plugin
       unique_str = unique_id.unpack('C*').map { |x| "%02x" % x }.join
       log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
 
+      start = Time.now
       begin
         begin
-          start = Time.now
           @client.import(database, table, UPLOAD_EXT, io, size, unique_str)
-        rescue TreasureData::NotFoundError => e
+        rescue TreasureData::NotFoundError
           unless @auto_create_table
-            raise e
+            raise
           end
           ensure_database_and_table(database, table)
           io.pos = 0
           retry
         end
+      rescue TreasureData::TooManyRequestsError
+        raise
+      rescue TreasureData::ClientError => e
+        raise Fluent::UnrecoverableError.new(e.message)
       rescue => e
         elapsed = Time.now - start
         ne = RuntimeError.new("Failed to upload to Treasure Data '#{database}.#{table}' table: #{e.inspect} (#{size} bytes; #{elapsed} seconds)")
@@ -269,6 +272,7 @@ module Fluent::Plugin
         @api_client.create_database(database)
         @api_client.create_log_table(database, table)
       rescue TreasureData::AlreadyExistsError
+        # ignored
       end
     end
   end

data/lib/fluent/plugin/td_plugin_version.rb

@@ -1,7 +1,7 @@
 module Fluent
   module Plugin
     module TreasureDataPlugin
-      VERSION = '1.1.0'
+      VERSION = '1.2.0'
     end
   end
 end

data/test/plugin/test_out_tdlog.rb

@@ -1,7 +1,7 @@
 require 'fluent/test'
 require 'fluent/test/driver/output'
 require 'fluent/plugin/out_tdlog'
-require 'test_helper.rb'
+require 'test_helper'
 
 class TreasureDataLogOutputTest < Test::Unit::TestCase
   TMP_DIR = File.dirname(__FILE__) + "/tmp"
@@ -144,7 +144,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     }
   end
 
-  def test_emit_with_time_symbole
+  def test_emit_with_time_symbol
     d = create_driver
     time, records = stub_seed_values
     database, table = d.instance.instance_variable_get(:@key).split(".", 2)
@@ -190,6 +190,38 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
     assert_equal 1, d.error_events.size
   end
 
+  def test_write_with_client_error
+    d = create_driver(DEFAULT_CONFIG)
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table, status: 400)
+
+    assert_nothing_raised(Fluent::UnrecoverableError) do
+      d.run(default_tag: 'test') {
+        records.each { |record|
+          d.feed(time, record)
+        }
+      }
+    end
+  end
+
+  def test_write_retry_if_too_many_requests
+    d = create_driver(DEFAULT_CONFIG)
+    time, records = stub_seed_values
+    database, table = d.instance.instance_variable_get(:@key).split(".", 2)
+    stub_td_table_create_request(database, table)
+    stub_td_import_request(stub_request_body(records, time), database, table, status: 429)
+
+    assert_raise(TreasureData::TooManyRequestsError) do
+      d.run(default_tag: 'test') {
+        records.each { |record|
+          d.feed(time, record)
+        }
+      }
+    end
+  end
+
   sub_test_case 'tag splitting for database and table' do
     def create_driver(conf = %[auto_create_table true])
       config = BASE_CONFIG + conf
@@ -197,7 +229,7 @@ class TreasureDataLogOutputTest < Test::Unit::TestCase
       Fluent::Test::Driver::Output.new(Fluent::Plugin::TreasureDataLogOutput).configure(config)
     end
 
-    data('evet_time' => 'event_time', 'int_time' => 'int')
+    data('event_time' => 'event_time', 'int_time' => 'int')
     def test_tag_split(time_class)
       d = create_driver
 

data/test/test_helper.rb

@@ -69,6 +69,7 @@ class Test::Unit::TestCase
     opts[:use_ssl] = true unless opts.has_key?(:use_ssl)
     format = opts[:format] || 'msgpack.gz'
     schema = opts[:use_ssl] ? 'https' : 'http'
+    status = opts[:status] || 200
     response = {"database" => db, "table" => table, "elapsed_time" => 0}.to_json
     endpoint = opts[:endpoint] ? opts[:endpoint] : TreasureData::API::DEFAULT_IMPORT_ENDPOINT
 
@@ -80,6 +81,6 @@ class Test::Unit::TestCase
     stub_request(:put, url_with_unique).with(:headers => {'Content-Type' => 'application/octet-stream'}) { |req|
       @auth_header = req.headers["Authorization"]
       stub_gzip_unwrap(req.body) == stub_gzip_unwrap(body)
-    }.to_return(:status => 200, :body => response)
+    }.to_return(:status => status, :body => response)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-td
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-18 00:00:00.000000000 Z
+date: 2022-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -34,16 +34,16 @@ dependencies:
   name: td-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.8
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -59,70 +59,98 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
-  name: webmock
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.5.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.8
+        version: 3.5.3
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.0.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 1.0.5
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.14.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.14.0
 description: Treasure Data Cloud Data Service plugin for Fluentd
 email: support@treasure-data.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
+- ".github/dependabot.yml"
+- ".github/workflows/codeql-analysis.yml"
 - ".gitignore"
-- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
-- Gemfile.msgpack.0.4
-- Gemfile.v0.12
+- Gemfile.lock
 - README.rdoc
 - Rakefile
 - example.conf
 - fluent-plugin-td.gemspec
 - lib/fluent/plugin/out_tdlog.rb
-- lib/fluent/plugin/td_plugin_util.rb
 - lib/fluent/plugin/td_plugin_version.rb
 - test/plugin/test_out_tdlog.rb
 - test/test_helper.rb
-homepage: http://www.treasuredata.com/
+homepage: https://www.treasuredata.com/
 licenses:
 - Apache-2.0
 metadata: {}
@@ -134,17 +162,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Treasure Data Cloud Data Service plugin for Fluentd
-test_files:
-- test/plugin/test_out_tdlog.rb
-- test/test_helper.rb
+test_files: []

data/.travis.yml

@@ -1,18 +0,0 @@
-rvm:
- - 2.1
- - 2.2.4
- - 2.3.1
- - 2.4.0
- - ruby-head
-
-gemfile:
- - Gemfile
-
-before_install: gem update bundler
-script: bundle exec rake test
-
-sudo: false
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head

data/Gemfile.msgpack.0.4

@@ -1,5 +0,0 @@
-source 'https://rubygems.org/'
-
-gem 'msgpack', '= 0.4.7'
-gem 'fluentd', '~> 0.10.55'
-gemspec

data/Gemfile.v0.12

@@ -1,6 +0,0 @@
-source "http://rubygems.org"
-
-gem 'json', '= 1.8.6'
-gem 'fluentd', '~> 0.12.0'
-
-gemspec

data/lib/fluent/plugin/td_plugin_util.rb

@@ -1,107 +0,0 @@
-module Fluent
-  module TDPluginUtil
-    require 'fileutils'
-    require 'stringio'
-    require 'tempfile'
-    require 'zlib'
-    require 'td-client'
-
-    def validate_database_and_table_name(database, table, conf)
-      begin
-        TreasureData::API.validate_database_name(database)
-      rescue => e
-        raise ConfigError, "Invalid database name #{database.inspect}: #{e}: #{conf}"
-      end
-      begin
-        TreasureData::API.validate_table_name(table)
-      rescue => e
-        raise ConfigError, "Invalid table name #{table.inspect}: #{e}: #{conf}"
-      end
-    end
-
-    def parse_bool_parameter(param)
-      if param.empty?
-        true
-      else
-        param = Config.bool_value(param)
-        raise ConfigError, "'true' or 'false' is required for #{key} option on tdlog output" if param.nil?
-        param
-      end
-    end
-
-    def summarize_record(record)
-      json = record.to_json
-      if json.size > 100
-        json[0..97] + "..."
-      else
-        json
-      end
-    end
-
-    def check_table_existence(database, table)
-      @table_list ||= {}
-      key = "#{database}.#{table}"
-      unless @table_list.has_key?(key)
-        log.debug "checking whether table '#{key}' exists on Treasure Data"
-        io = StringIO.new(@empty_gz_data)
-        begin
-          # here doesn't check whether target table is item table or not because import-only user can't read the table status.
-          # So I use empty import request to check table existence.
-          @client.import(database, table, "msgpack.gz", io, io.size)
-          @table_list[key] = true
-        rescue TreasureData::NotFoundError
-          args = self.class == TreasureDataItemOutput ? ' -t item' : ''
-          raise "Table #{key.inspect} does not exist on Treasure Data. Use 'td table:create #{database} #{table}#{args}' to create it."
-        rescue => e
-          log.warn "failed to check table existence on Treasure Data", :error => e.inspect
-          log.debug_backtrace e
-        end
-      end
-    end
-
-    def write(chunk)
-      unique_id = chunk.unique_id
-      database, table = chunk.key.split('.', 2)
-
-      FileUtils.mkdir_p(@tmpdir) unless @tmpdir.nil?
-      f = Tempfile.new(@tmpdir_prefix, @tmpdir)
-      w = Zlib::GzipWriter.new(f)
-
-      chunk.write_to(w)
-      w.finish
-      w = nil
-
-      size = f.pos
-      f.pos = 0
-      upload(database, table, f, size, unique_id)
-    ensure
-      w.close if w
-      f.close if f
-    end
-
-    # assume @client and @auto_create_table variable exist
-    def upload(database, table, io, size, unique_id)
-      unique_str = unique_id.unpack('C*').map {|x| "%02x" % x }.join
-      log.trace { "uploading logs to Treasure Data database=#{database} table=#{table} (#{size}bytes)" }
-
-      begin
-        begin
-          start = Time.now
-          @client.import(database, table, "msgpack.gz", io, size, unique_str)
-        rescue TreasureData::NotFoundError => e
-          unless @auto_create_table
-            raise e
-          end
-          ensure_database_and_table(database, table)
-          io.pos = 0
-          retry
-        end
-      rescue => e
-        elapsed = Time.now - start
-        ne = RuntimeError.new("Failed to upload to TreasureData: #{e} (#{size} bytes; #{elapsed} seconds)")
-        ne.set_backtrace(e.backtrace)
-        raise ne
-      end
-    end
-  end
-end