logstash-output-treasure_data 0.9.2 → 1.0.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3712df2d7d14d7849433f8e6411093df9d694c64
-  data.tar.gz: 1c886ecf4d3b728f2bb4248c124fe42ed474de3f
+  metadata.gz: 1c3ff3748f4a47764ad43b4dca3f864f1b50f93b
+  data.tar.gz: 6bd09e966b142576773092f4d301598c5da15404
 SHA512:
-  metadata.gz: 082c619cec077b8426b4aabfabab2802e2f12001b0b3ca1773704209cd1dbabf98610b821847d1731697d526520080e7449b6668932be3d4489a83a18548bc20
-  data.tar.gz: da8a0361e01b1badd4feacfd5a78ed1b81c271181a4b3981572aaa828333bbfe4614d8c61914fb2e700f3398b4f9183eb02a5e9f2f74b746b35375a0e8b41921
+  metadata.gz: 82344fc02835f4a311dc33b39b7a837d3e0add9c3f1a8ad2e826013c44ff6274e6427f89b4901aeaa3df2d1ff8bd694ae186602a642d48b491a3293ba40f94e7
+  data.tar.gz: c13901a2a622a6d9cd8908f1113a03f8ef0210c4b73679f8106c6a9d7b4a3289d0b370f79755bcbe617c1e1967489fbd3413da21c94cd60737a975760f2a4193

data/lib/logstash/outputs/treasure_data.rb

@@ -6,27 +6,18 @@ require "stud/buffer"
 
 require "td-client"
 require "msgpack"
-require "uuid"
 
+require "securerandom"
 require "stringio"
 require "zlib"
 
 # Logstash output plugin to send data to Treasure Data service.
 
-class LogStash::Event
-  def to_msgpack(packer=nil)
-    # LogStash objects (ex: LogStash::Timestamp) are impossible to serialize by msgpack
-    begin
-      @data.reject{|a,b| a == TIMESTAMP }.to_msgpack
-    rescue ArgumentError, NoMethodError
-      LogStash::Json.load(LogStash::Json.dump(@data)).to_msgpack
-    end
-  end
-end
-
 class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
   include Stud::Buffer
 
+  concurrency :shared if self.respond_to?(:concurrency)
+
   config_name "treasure_data"
 
   IMPORT_SIZE_LIMIT = 32 * 1024 * 1024
@@ -58,12 +49,11 @@ class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
   def register
     @empty_gz_data = TreasureData::API.create_empty_gz_data
     @user_agent = "logstash-output-treasure_data: #{VERSION}".freeze
-    @uuid = UUID.new
 
     TreasureData::API.validate_database_name(@database)
     TreasureData::API.validate_table_name(@table)
 
-    client_opts = {
+    @client_opts = {
       ssl: @use_ssl,
       http_proxy: @http_proxy,
       user_agent: @user_agent,
@@ -72,7 +62,6 @@ class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
       read_timeout: @read_timeout,
       send_timeout: @send_timeout
     }
-    @client = TreasureData::Client.new(@apikey.value, client_opts)
 
     buffer_initialize(
       max_items: @flush_size,
@@ -83,15 +72,16 @@ class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
 
   public
   def receive(event)
+    # Logstash::Output::Base#multi_receive(events) calls this method by `events.each{|event| receive(event) }`
     record = event.clone
     @logger.debug "receive a event"
 
-    record['time'] ||= (record.timestamp.to_i || Time.now.to_i)
+    record.set('time', record.timestamp.to_i || Time.now.to_i) unless record.get('time')
     if record.to_hash.size > RECORD_KEYS_LIMIT
       raise "Too many number of keys (#{record.keys.size} keys)"
     end
 
-    row = record.to_msgpack
+    row = convert_to_msgpack(record)
     if row.bytesize > RECORD_SIZE_LIMIT
       raise "Too large record (#{row.bytesize} bytes with keys:#{record.keys.join(',')})"
     end
@@ -100,12 +90,18 @@ class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
     @logger.debug "buffered a event"
   end
 
+  def convert_to_msgpack(record)
+    record.to_hash.to_msgpack
+  rescue ArgumentError, NoMethodError
+    LogStash::Json.load(LogStash::Json.dump(record.to_hash)).to_msgpack
+  end
+
   public
   def flush(events, teardown = false)
     @logger.debug "flushing #{events} events (may include chunk uuid)"
     return if events.size < 1
     if UUID_FORMAT !~ events.first
-      new_uuid = @uuid.generate
+      new_uuid = SecureRandom.uuid
       @logger.debug "adding chunk uuid #{new_uuid}"
       events.unshift new_uuid
     end
@@ -122,15 +118,16 @@ class LogStash::Outputs::TreasureData < LogStash::Outputs::Base
       f.finish
     }
     data = io.string
+    client = TreasureData::Client.new(@apikey.value, @client_opts)
     @logger.debug "sending gzipped chunk #{uuid}, #{data.bytesize} bytes"
     begin
-      @client.import(@database, @table, "msgpack.gz", data, data.bytesize, uuid)
+      client.import(@database, @table, "msgpack.gz", data, data.bytesize, uuid)
 
     rescue TreasureData::NotFoundError => e
       raise unless @auto_create_table
 
       @logger.info "creating missing table #{@table} on database #{@database} for chunk #{uuid}"
-      ensure_table_exists(@client, @database, @table)
+      ensure_table_exists(client, @database, @table)
       @logger.info "retrying upload chunk #{uuid}"
       retry
     end

data/logstash-output-treasure_data.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-treasure_data'
-  s.version     = "0.9.2"
+  s.version     = "1.0.0.pre1"
   s.licenses    = ["Apache License (2.0)"]
   s.summary     = "Logstash plugin to store data into Treasure Data service."
   s.description = "This gem is a logstash plugin to store data from logstash to Treasure Data service. This gem is not a stand-alone program."
@@ -18,10 +18,9 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency "logstash-codec-plain"
-  s.add_runtime_dependency "msgpack", "~> 0.7.4"
-  s.add_runtime_dependency "td-client", "~> 0.8.78"
-  s.add_runtime_dependency "uuid", "~> 2.3"
+  s.add_runtime_dependency "msgpack", "~> 1.0"
+  s.add_runtime_dependency "td-client", "~> 1.0"
   s.add_development_dependency "logstash-devutils"
 end

metadata

@@ -1,107 +1,92 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-treasure_data
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 1.0.0.pre1
 platform: ruby
 authors:
 - Satoshi Tagomori
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-25 00:00:00.000000000 Z
+date: 2017-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
-  type: :runtime
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.4
-  type: :runtime
+        version: '1.0'
+  name: msgpack
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.7.4
-- !ruby/object:Gem::Dependency
-  name: td-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.8.78
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.78
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: uuid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
-  type: :runtime
+        version: '1.0'
+  name: td-client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.3'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a logstash plugin to store data from logstash to Treasure
-  Data service. This gem is not a stand-alone program.
+description: This gem is a logstash plugin to store data from logstash to Treasure Data service. This gem is not a stand-alone program.
 email: tagomoris@gmail.com
 executables: []
 extensions: []
@@ -123,7 +108,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -134,15 +119,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
 specification_version: 4
 summary: Logstash plugin to store data into Treasure Data service.
 test_files:
 - spec/outputs/treasure_data_spec.rb
-has_rdoc: