fluent-plugin-kinesis 0.4.1 → 1.0.0

data/lib/kinesis_producer/protobuf/config.pb.rb

@@ -0,0 +1,66 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+require 'protobuf/message'
+
+module KinesisProducer
+  module Protobuf
+    ##
+    # Message Classes
+    #
+    class AdditionalDimension < ::Protobuf::Message; end
+    class Configuration < ::Protobuf::Message; end
+
+
+    ##
+    # Message Fields
+    #
+    class AdditionalDimension
+      required :string, :key, 1
+      required :string, :value, 2
+      required :string, :granularity, 3
+    end
+
+    class Configuration
+      repeated ::KinesisProducer::Protobuf::AdditionalDimension, :additional_metric_dims, 128
+      optional :bool, :aggregation_enabled, 1, :default => true
+      optional :uint64, :aggregation_max_count, 2, :default => 4294967295
+      optional :uint64, :aggregation_max_size, 3, :default => 51200
+      optional :uint64, :collection_max_count, 4, :default => 500
+      optional :uint64, :collection_max_size, 5, :default => 5242880
+      optional :uint64, :connect_timeout, 6, :default => 6000
+      optional :string, :custom_endpoint, 7
+      optional :bool, :fail_if_throttled, 8, :default => false
+      optional :string, :log_level, 9, :default => "info"
+      optional :uint64, :max_connections, 10, :default => 24
+      optional :string, :metrics_granularity, 11, :default => "shard"
+      optional :string, :metrics_level, 12, :default => "detailed"
+      optional :string, :metrics_namespace, 13, :default => "KinesisProducerLibrary"
+      optional :uint64, :metrics_upload_delay, 14, :default => 60000
+      optional :uint64, :min_connections, 15, :default => 1
+      optional :uint64, :port, 16, :default => 443
+      optional :uint64, :rate_limit, 17, :default => 150
+      optional :uint64, :record_max_buffered_time, 18, :default => 100
+      optional :uint64, :record_ttl, 19, :default => 30000
+      optional :string, :region, 20
+      optional :uint64, :request_timeout, 21, :default => 6000
+      optional :bool, :verify_certificate, 22, :default => true
+    end
+
+  end
+end
+

data/lib/kinesis_producer/protobuf/messages.pb.rb

@@ -0,0 +1,151 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+require 'protobuf/message'
+
+
+##
+# Imports
+#
+require 'kinesis_producer/protobuf/config.pb'
+
+module KinesisProducer
+  module Protobuf
+
+    ##
+    # Message Classes
+    #
+    class Tag < ::Protobuf::Message; end
+    class Record < ::Protobuf::Message; end
+    class AggregatedRecord < ::Protobuf::Message; end
+    class Message < ::Protobuf::Message; end
+    class PutRecord < ::Protobuf::Message; end
+    class Flush < ::Protobuf::Message; end
+    class Attempt < ::Protobuf::Message; end
+    class PutRecordResult < ::Protobuf::Message; end
+    class Credentials < ::Protobuf::Message; end
+    class SetCredentials < ::Protobuf::Message; end
+    class Dimension < ::Protobuf::Message; end
+    class Stats < ::Protobuf::Message; end
+    class Metric < ::Protobuf::Message; end
+    class MetricsRequest < ::Protobuf::Message; end
+    class MetricsResponse < ::Protobuf::Message; end
+
+
+    ##
+    # Message Fields
+    #
+    class Tag
+      required :string, :key, 1
+      optional :string, :value, 2
+    end
+
+    class Record
+      required :uint64, :partition_key_index, 1
+      optional :uint64, :explicit_hash_key_index, 2
+      required :bytes, :data, 3
+      repeated ::KinesisProducer::Protobuf::Tag, :tags, 4
+    end
+
+    class AggregatedRecord
+      repeated :string, :partition_key_table, 1
+      repeated :string, :explicit_hash_key_table, 2
+      repeated ::KinesisProducer::Protobuf::Record, :records, 3
+    end
+
+    class Message
+      required :uint64, :id, 1
+      optional :uint64, :source_id, 2
+      optional ::KinesisProducer::Protobuf::PutRecord, :put_record, 3
+      optional ::KinesisProducer::Protobuf::Flush, :flush, 4
+      optional ::KinesisProducer::Protobuf::PutRecordResult, :put_record_result, 5
+      optional ::KinesisProducer::Protobuf::Configuration, :configuration, 6
+      optional ::KinesisProducer::Protobuf::MetricsRequest, :metrics_request, 7
+      optional ::KinesisProducer::Protobuf::MetricsResponse, :metrics_response, 8
+      optional ::KinesisProducer::Protobuf::SetCredentials, :set_credentials, 9
+    end
+
+    class PutRecord
+      required :string, :stream_name, 1
+      required :string, :partition_key, 2
+      optional :string, :explicit_hash_key, 3
+      required :bytes, :data, 4
+    end
+
+    class Flush
+      optional :string, :stream_name, 1
+    end
+
+    class Attempt
+      required :uint32, :delay, 1
+      required :uint32, :duration, 2
+      required :bool, :success, 3
+      optional :string, :error_code, 4
+      optional :string, :error_message, 5
+    end
+
+    class PutRecordResult
+      repeated ::KinesisProducer::Protobuf::Attempt, :attempts, 1
+      required :bool, :success, 2
+      optional :string, :shard_id, 3
+      optional :string, :sequence_number, 4
+    end
+
+    class Credentials
+      required :string, :akid, 1
+      required :string, :secret_key, 2
+      optional :string, :token, 3
+    end
+
+    class SetCredentials
+      optional :bool, :for_metrics, 1
+      required ::KinesisProducer::Protobuf::Credentials, :credentials, 2
+    end
+
+    class Dimension
+      required :string, :key, 1
+      required :string, :value, 2
+    end
+
+    class Stats
+      required :double, :count, 1
+      required :double, :sum, 2
+      required :double, :mean, 3
+      required :double, :min, 4
+      required :double, :max, 5
+    end
+
+    class Metric
+      required :string, :name, 1
+      repeated ::KinesisProducer::Protobuf::Dimension, :dimensions, 2
+      required ::KinesisProducer::Protobuf::Stats, :stats, 3
+      required :uint64, :seconds, 4
+    end
+
+    class MetricsRequest
+      optional :string, :name, 1
+      optional :uint64, :seconds, 2
+    end
+
+    class MetricsResponse
+      repeated ::KinesisProducer::Protobuf::Metric, :metrics, 1
+    end
+
+  end
+
+end
+

data/lib/kinesis_producer/tasks/binary.rake

@@ -0,0 +1,73 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'kinesis_producer'
+require 'net/http'
+require 'zip'
+
+jar_version = "0.10.2"
+jar_file = "amazon-kinesis-producer-#{jar_version}.jar"
+jar_url = "https://search.maven.org/remotecontent?filepath=com/amazonaws/amazon-kinesis-producer/#{jar_version}/#{jar_file}"
+cache_dir = Pathname.new(".cache")
+cache_jar_file = cache_dir.join(jar_file)
+binaries = KinesisProducer::Binary::Files.values
+
+directory cache_dir
+
+file cache_jar_file => [cache_dir] do |t|
+  puts "Downloading #{jar_file}"
+  download(jar_url, t.name)
+end
+
+binaries.each do |bin|
+  file bin => [cache_jar_file] do |t|
+    puts "Extracting #{bin} from #{jar_file}"
+    unzip(cache_jar_file, bin)
+    chmod 0755, bin
+  end
+end
+
+task :binaries => binaries
+
+task :clean do
+  rm_rf cache_dir
+  rm_rf File.dirname(File.dirname(binary))
+end
+
+def download(url, target)
+  rm_f target
+  uri = URI(url)
+  Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
+    req = Net::HTTP::Get.new(uri.request_uri)
+    https.request(req) do |res|
+      open(target, 'wb') do |io|
+        res.read_body do |chunk|
+          io.write(chunk)
+        end
+      end
+    end
+  end
+end
+
+def unzip(zip_file, *targets)
+  rm_f targets
+  Zip::File.open(zip_file) do |z|
+    z.each do |entry|
+      if targets.include?(entry.name)
+        mkdir_p File.dirname(entry.name)
+        entry.extract(entry.name)
+      end
+    end
+  end
+end

metadata

@@ -1,29 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-17 00:00:00.000000000 Z
+date: 2016-03-18 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.58
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.5.5
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: os
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -38,74 +114,132 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
-    - - "<"
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.53
-    - - "<"
+        version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 0.4.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.9.2
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: net-empty_port
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
-    - - "<"
+        version: 0.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: dummer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.12
-    - - "<"
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.8
-  type: :runtime
+        version: 1.1.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.8
+        version: 1.1.0
 description: 
 email: 
 executables: []
@@ -118,17 +252,45 @@ files:
 - CONTRIBUTORS.txt
 - Gemfile
 - LICENSE.txt
+- Makefile
 - NOTICE.txt
+- README-v0.4.md
 - README.md
 - Rakefile
+- amazon-kinesis-producer-native-binaries/linux/kinesis_producer
+- amazon-kinesis-producer-native-binaries/osx/kinesis_producer
+- amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
+- benchmark/dummer.conf
+- benchmark/firehose.conf
+- benchmark/producer.conf
+- benchmark/streams.conf
 - fluent-plugin-kinesis.gemspec
+- gemfiles/Gemfile.fluentd-0.10.58
+- lib/fluent/plugin/kinesis_helper.rb
+- lib/fluent/plugin/kinesis_helper/api.rb
+- lib/fluent/plugin/kinesis_helper/class_methods.rb
+- lib/fluent/plugin/kinesis_helper/client.rb
+- lib/fluent/plugin/kinesis_helper/credentials.rb
+- lib/fluent/plugin/kinesis_helper/error.rb
+- lib/fluent/plugin/kinesis_helper/format.rb
+- lib/fluent/plugin/kinesis_helper/initialize.rb
+- lib/fluent/plugin/kinesis_helper/kpl.rb
 - lib/fluent/plugin/out_kinesis.rb
-- lib/fluent/plugin/version.rb
-- test/helper.rb
-- test/plugin/test_out_kinesis.rb
+- lib/fluent/plugin/out_kinesis_firehose.rb
+- lib/fluent/plugin/out_kinesis_producer.rb
+- lib/fluent/plugin/out_kinesis_streams.rb
+- lib/fluent/plugin/patched_detach_process_impl.rb
+- lib/fluent_plugin_kinesis/version.rb
+- lib/kinesis_producer.rb
+- lib/kinesis_producer/binary.rb
+- lib/kinesis_producer/daemon.rb
+- lib/kinesis_producer/library.rb
+- lib/kinesis_producer/protobuf/config.pb.rb
+- lib/kinesis_producer/protobuf/messages.pb.rb
+- lib/kinesis_producer/tasks/binary.rake
 homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
 licenses:
-- Apache License, Version 2.0
+- Amazon Software License
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -138,7 +300,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -150,6 +312,4 @@ rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.
-test_files:
-- test/helper.rb
-- test/plugin/test_out_kinesis.rb
+test_files: []