td-client 0.8.68 → 0.8.69

data/lib/td/client/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
   class Client
-    VERSION = '0.8.68'
+    VERSION = '0.8.69'
   end
 end

data/spec/spec_helper.rb

@@ -1,13 +1,21 @@
 require 'rubygems'
 
-if ENV['SIMPLE_COV']
-  begin
+begin
+  if defined?(:RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+    # SimpleCov officially supports MRI 1.9+ only for now
+    # https://github.com/colszowka/simplecov#ruby-version-compatibility
+
     require 'simplecov'
-    SimpleCov.start do
-      add_filter '/spec/'
-    end
-  rescue LoadError
+    require 'coveralls'
+
+    SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+      SimpleCov::Formatter::HTMLFormatter,
+      Coveralls::SimpleCov::Formatter
+    ]
+    SimpleCov.start("test_frameworks")
   end
+rescue NameError
+  # skip measuring coverage at Ruby 1.8
 end
 
 require 'rspec'

data/spec/td/client/bulk_load_spec.rb

@@ -0,0 +1,358 @@
+require 'spec_helper'
+require 'td/client/spec_resources'
+require 'tempfile'
+
+describe 'BulkImport API' do
+  include_context 'spec symbols'
+  include_context 'common helper'
+
+  let :api do
+    Client.new(nil, {:max_cumul_retry_delay => -1})
+  end
+
+  let :retry_api do
+    API.new(nil, {:retry_delay => 1, :max_cumul_retry_delay => 1})
+  end
+
+  let :original_config do
+    {
+      :config => {
+        :type => "s3_file",
+        :access_key_id => "key",
+        :secret_access_key => "secret",
+        :endpoint => "s3.amazonaws.com",
+        :bucket => "td-bulk-loader-test-tokyo",
+        :path_prefix => "in/nahi/sample"
+      }
+    }
+  end
+
+  let :guessed_config do
+    {
+      "config" => {
+        "type" => "s3_file",
+        "access_key_id" => "key",
+        "secret_access_key" => "secret",
+        "endpoint" => "s3.amazonaws.com",
+        "bucket" => "td-bulk-loader-test-tokyo",
+        "path_prefix" => "in/nahi/sample",
+        "parser" => {
+          "charset" => "UTF-8",
+          "newline" => "LF",
+          "type" => "csv",
+          "delimiter" => ",",
+          "header_line" => false,
+          "columns" => [
+            {"name" => "time", "type" => "long"},
+            {"name" => "c1", "type" => "long"},
+            {"name" => "c2", "type" => "string"},
+            {"name" => "c3", "type" => "string"},
+          ]
+        },
+        "decoders" => [
+          {"type" => "gzip"}
+        ]
+      }
+    }
+  end
+
+  let :preview_result do
+    {
+      "schema" => [
+        {"index" => 0, "name" => "c0", "type" => "string"},
+        {"index" => 1, "name" => "c1", "type" => "long"},
+        {"index" => 2, "name" => "c2", "type" => "string"},
+        {"index" => 3, "name" => "c3", "type" => "string"}
+      ],
+      "records" => [
+        ["19920116", 32864, "06612", "00195"],
+        ["19910729", 14824, "07706", "00058"],
+        ["19950708", 27559, "03244", "00034"],
+        ["19931010", 11270, "03459", "00159"],
+        ["19981117", 20461, "01409", "00128"],
+        ["19981117", 20461, "00203", "00128"],
+        ["19930108", 44402, "01489", "00001"],
+        ["19960104", 16528, "04848", "00184"],
+        ["19960104", 16528, "01766", "00184"],
+        ["19881022", 26114, "06960", "00175"]
+      ]
+    }
+  end
+
+  let :bulk_load_session do
+    guessed_config.dup.merge(
+      {
+        "name" => "nahi_test_1",
+        "cron" => "@daily",
+        "timezone" => "Asia/Tokyo",
+        "delay" => 3600
+      }
+    )
+  end
+
+  let :bulk_load_job do
+    guessed_config.dup.merge(
+      {
+        "job_id" => 123456,
+        "account_id" => 1,
+        "status" => "success",
+        "records" => 10,
+        "schema" => [["c0", "string", ""], ["c1", "long", ""], ["c2", "string", ""], ["c3", "string", ""]],
+        "database" => {"id" => 189263, "name" => "nahidb"},
+        "table" => {"id" => 176281, "name" => "bulkload_import_test"},
+        "created_at" => 1426738133,
+        "updated_at" => 1426738145,
+        "start_at" => 1426738134,
+        "end_at" => 1426738144
+      }
+    )
+  end
+
+  describe 'guess' do
+    it 'returns guessed json' do
+      stub_api_request(:post, '/v3/bulk_loads/guess').
+        with(:body => original_config.to_json).
+        to_return(:body => guessed_config.to_json)
+      api.bulk_load_guess(
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(original_config)
+      ).to_h.should == guessed_config
+    end
+
+    it 'raises an error' do
+      stub_api_request(:post, '/v3/bulk_loads/guess').
+        with(:body => original_config.to_json).
+        to_return(:status => 500, :body => guessed_config.to_json)
+      expect {
+        api.bulk_load_guess(
+          TreasureData::API::BulkLoad::BulkLoad.from_hash(original_config)
+        )
+      }.to raise_error(TreasureData::APIError)
+    end
+
+    it 'raises on validation error' do
+      config = TreasureData::API::BulkLoad::BulkLoad.from_hash({})
+      expect {
+        api.bulk_load_guess(config)
+      }.to raise_error(ArgumentError)
+    end
+
+    it 'raises on nested validation error' do
+      config = TreasureData::API::BulkLoad::BulkLoad.from_hash(original_config)
+      config.config.type = nil
+      expect {
+        api.bulk_load_guess(config)
+      }.to raise_error(ArgumentError)
+    end
+
+    it 'perform redo on 500 error' do
+      stub_api_request(:post, '/v3/bulk_loads/guess').
+        with(:body => original_config.to_json).
+        to_return(:status => 500, :body => guessed_config.to_json)
+      begin
+        retry_api.bulk_load_guess(
+          TreasureData::API::BulkLoad::BulkLoad.from_hash(original_config)
+        ).should != nil
+      rescue TreasureData::APIError => e
+        e.message.should =~ /^500: BulkLoad configuration guess failed/
+      end
+    end
+
+    it 'perform retries on connection failure' do
+      api = retry_api
+      api.instance_eval { @api }.stub(:post).and_raise(SocketError.new('>>'))
+      begin
+        retry_api.bulk_load_guess(
+          TreasureData::API::BulkLoad::BulkLoad.from_hash(original_config)
+        )
+      rescue SocketError => e
+        e.message.should == '>> (Retried 1 times in 1 seconds)'
+      end
+    end
+  end
+
+  describe 'preview' do
+    it 'returns preview json' do
+      stub_api_request(:post, '/v3/bulk_loads/preview').
+        with(:body => guessed_config.to_json).
+        to_return(:body => preview_result.to_json)
+      api.bulk_load_preview(
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config)
+      ).to_h.should == preview_result
+    end
+
+    it 'raises an error' do
+      stub_api_request(:post, '/v3/bulk_loads/preview').
+        with(:body => guessed_config.to_json).
+        to_return(:status => 500, :body => preview_result.to_json)
+      expect {
+        api.bulk_load_preview(
+          TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config)
+        )
+      }.to raise_error(TreasureData::APIError)
+    end
+
+    it 'raises on validation error' do
+      config = TreasureData::API::BulkLoad::BulkLoad.from_hash({})
+      expect {
+        api.bulk_load_preview(config)
+      }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe 'issue' do
+    it 'returns job id' do
+      expected_request = guessed_config.dup
+      expected_request['database'] = 'database'
+      expected_request['table'] = 'table'
+      stub_api_request(:post, '/v3/job/issue/bulkload/database').
+        with(:body => expected_request.to_json).
+        to_return(:body => {'job_id' => 12345}.to_json)
+      api.bulk_load_issue(
+        'database',
+        'table',
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config)
+      ).should == '12345'
+    end
+
+    it 'raises on validation error' do
+      config = TreasureData::API::BulkLoad::BulkLoad.from_hash({})
+      expect {
+        api.bulk_load_issue(config)
+      }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe 'list' do
+    it 'returns BulkLoadSession' do
+      stub_api_request(:get, '/v3/bulk_loads').
+        to_return(:body => [bulk_load_session, bulk_load_session].to_json)
+      result = api.bulk_load_list
+      result.size.should == 2
+      result.first.to_h.should == bulk_load_session
+    end
+
+    it 'returns empty' do
+      stub_api_request(:get, '/v3/bulk_loads').
+        to_return(:body => [].to_json)
+      api.bulk_load_list.size.should == 0
+    end
+  end
+
+  describe 'create' do
+    it 'returns registered bulk_load_session' do
+      expected_request = guessed_config.dup
+      expected_request['name'] = 'nahi_test_1'
+      expected_request['cron'] = '@daily'
+      expected_request['timezone'] = 'Asia/Tokyo'
+      expected_request['delay'] = 3600
+      expected_request['database'] = 'database'
+      expected_request['table'] = 'table'
+      stub_api_request(:post, '/v3/bulk_loads').
+        with(:body => expected_request.to_json).
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_create(
+        'nahi_test_1',
+        'database',
+        'table',
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config),
+        {
+          cron: '@daily',
+          timezone: 'Asia/Tokyo',
+          delay: 3600
+        }
+      ).to_h.should == bulk_load_session
+    end
+
+    it 'accepts empty option' do
+      expected_request = guessed_config.dup
+      expected_request['name'] = 'nahi_test_1'
+      expected_request['database'] = 'database'
+      expected_request['table'] = 'table'
+      stub_api_request(:post, '/v3/bulk_loads').
+        with(:body => expected_request.to_json).
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_create(
+        'nahi_test_1',
+        'database',
+        'table',
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config)
+      ).to_h.should == bulk_load_session
+    end
+
+    it 'accepts time_column option' do
+      expected_request = guessed_config.dup
+      expected_request['name'] = 'nahi_test_1'
+      expected_request['time_column'] = 'c0'
+      expected_request['database'] = 'database'
+      expected_request['table'] = 'table'
+      stub_api_request(:post, '/v3/bulk_loads').
+        with(:body => expected_request.to_json).
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_create(
+        'nahi_test_1',
+        'database',
+        'table',
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(guessed_config),
+        {
+          time_column: 'c0'
+        }
+      ).to_h.should == bulk_load_session
+    end
+  end
+
+  describe 'show' do
+    it 'returns bulk_load_session' do
+      stub_api_request(:get, '/v3/bulk_loads/nahi_test_1').
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_show('nahi_test_1').to_h.should == bulk_load_session
+    end
+  end
+
+  describe 'update' do
+    it 'returns updated bulk_load_session' do
+      stub_api_request(:put, '/v3/bulk_loads/nahi_test_1').
+        with(:body => bulk_load_session.to_json).
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_update(
+        'nahi_test_1',
+        TreasureData::API::BulkLoad::BulkLoad.from_hash(bulk_load_session)
+      ).to_h.should == bulk_load_session
+    end
+  end
+
+  describe 'delete' do
+    it 'returns updated bulk_load_session' do
+      stub_api_request(:delete, '/v3/bulk_loads/nahi_test_1').
+        to_return(:body => bulk_load_session.to_json)
+      api.bulk_load_delete('nahi_test_1').to_h.should == bulk_load_session
+    end
+  end
+
+  describe 'history' do
+    it 'returns list of jobs' do
+      stub_api_request(:get, '/v3/bulk_loads/nahi_test_1/jobs').
+        to_return(:body => [bulk_load_job, bulk_load_job].to_json)
+      result = api.bulk_load_history('nahi_test_1')
+      result.size.should == 2
+      result.first.to_h.should == bulk_load_job
+    end
+  end
+
+  describe 'run' do
+    it 'returns job_id' do
+      stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
+        with(:body => '').
+        to_return(:body => {'job_id' => 12345}.to_json)
+      api.bulk_load_run('nahi_test_1').should == '12345'
+    end
+
+    it 'accepts scheduled_time' do
+      now = Time.now.to_i
+      stub_api_request(:post, '/v3/bulk_loads/nahi_test_1/jobs').
+        with(:body => "scheduled_time=#{now}").
+        to_return(:body => {'job_id' => 12345}.to_json)
+      api.bulk_load_run('nahi_test_1', now).should == '12345'
+    end
+  end
+
+end

data/spec/td/client/job_api_spec.rb

@@ -241,14 +241,16 @@ describe 'Job API' do
 
   describe 'job_result_each_with_compr_size' do
     let :packed do
-      s = StringIO.new
-      Zlib::GzipWriter.wrap(s) do |f|
-        pk = MessagePack::Packer.new(f)
-        pk.write('hello')
-        pk.write('world')
-        pk.flush
-      end
-      s.string
+      # Hard code fixture data to make the size stable
+      # s = StringIO.new
+      # Zlib::GzipWriter.wrap(s) do |f|
+      #   pk = MessagePack::Packer.new(f)
+      #   pk.write('hello')
+      #   pk.write('world')
+      #   pk.flush
+      # end
+      # s.string
+      "\u001F\x8B\b\u0000#\xA1\x93T\u0000\u0003[\x9A\x91\x9A\x93\x93\xBF\xB4<\xBF('\u0005\u0000e 0\xB3\f\u0000\u0000\u0000"
     end
 
     it 'yields job result for each row with progress' do

data/spec/td/client_sched_spec.rb

@@ -33,7 +33,7 @@ describe 'Schedule Command' do
         to_return(:body => {'count' => 1, 'history' => [h]}.to_json)
 
       client.history(sched_name, 0, 19).each do |scheduled_job|
-        scheduled_job.scheduled_at.xmlschema.should == '2015-02-17T14:16:00+09:00'
+        scheduled_job.scheduled_at.xmlschema.should == Time.parse('2015-02-17T14:16:00+09:00').xmlschema #avoid depending on CI's Locale
         scheduled_job.job_id.should == 'job_id'
         scheduled_job.status.should == 'status'
         scheduled_job.priority.should == 'priority'

metadata

@@ -1,147 +1,203 @@
 --- !ruby/object:Gem::Specification
 name: td-client
 version: !ruby/object:Gem::Version
-  version: 0.8.68
+  version: 0.8.69
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-17 00:00:00.000000000 Z
+date: 2015-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.3
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.4
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.0
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.1
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.2
-    - - '!='
+    - - "!="
       - !ruby/object:Gem::Version
         version: 0.5.3
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.7.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.7.6
 - !ruby/object:Gem::Dependency
   name: httpclient
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '1.25'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.16'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Treasure Data API library for Ruby
@@ -150,9 +206,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- data/ca-bundle.crt
+- lib/td-client.rb
+- lib/td/client.rb
+- lib/td/client/api.rb
 - lib/td/client/api/access_control.rb
 - lib/td/client/api/account.rb
 - lib/td/client/api/bulk_import.rb
+- lib/td/client/api/bulk_load.rb
 - lib/td/client/api/database.rb
 - lib/td/client/api/export.rb
 - lib/td/client/api/import.rb
@@ -162,22 +223,20 @@ files:
 - lib/td/client/api/schedule.rb
 - lib/td/client/api/server_status.rb
 - lib/td/client/api/table.rb
+- lib/td/client/api/to_hash_struct.rb
 - lib/td/client/api/user.rb
-- lib/td/client/api.rb
 - lib/td/client/api_error.rb
 - lib/td/client/compat_gzip_reader.rb
 - lib/td/client/model.rb
 - lib/td/client/version.rb
-- lib/td/client.rb
 - lib/td/core_ext/openssl/ssl/sslcontext/set_params.rb
-- lib/td-client.rb
-- data/ca-bundle.crt
 - spec/spec_helper.rb
 - spec/td/client/access_control_api_spec.rb
 - spec/td/client/account_api_spec.rb
 - spec/td/client/api_spec.rb
 - spec/td/client/api_ssl_connection_spec.rb
 - spec/td/client/bulk_import_spec.rb
+- spec/td/client/bulk_load_spec.rb
 - spec/td/client/db_api_spec.rb
 - spec/td/client/export_api_spec.rb
 - spec/td/client/import_api_spec.rb
@@ -199,17 +258,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Treasure Data API library for Ruby
@@ -219,6 +278,7 @@ test_files:
 - spec/td/client/api_spec.rb
 - spec/td/client/api_ssl_connection_spec.rb
 - spec/td/client/bulk_import_spec.rb
+- spec/td/client/bulk_load_spec.rb
 - spec/td/client/db_api_spec.rb
 - spec/td/client/export_api_spec.rb
 - spec/td/client/import_api_spec.rb
@@ -230,3 +290,4 @@ test_files:
 - spec/td/client/table_api_spec.rb
 - spec/td/client/user_api_spec.rb
 - spec/td/client_sched_spec.rb
+has_rdoc: false