fluent-plugin-kinesis 1.3.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +2 -23
  3. data/CHANGELOG.md +13 -0
  4. data/Gemfile +9 -9
  5. data/LICENSE.txt +201 -40
  6. data/Makefile +24 -31
  7. data/README.md +179 -308
  8. data/Rakefile +9 -13
  9. data/benchmark/task.rake +96 -58
  10. data/fluent-plugin-kinesis.gemspec +15 -19
  11. data/gemfiles/Gemfile.fluentd-0.12 +10 -10
  12. data/lib/fluent/plugin/kinesis.rb +166 -0
  13. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +99 -0
  14. data/lib/fluent/plugin/kinesis_helper/api.rb +152 -121
  15. data/lib/fluent/plugin/kinesis_helper/client.rb +125 -12
  16. data/lib/fluent/plugin/out_kinesis_firehose.rb +40 -27
  17. data/lib/fluent/plugin/out_kinesis_streams.rb +51 -30
  18. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +76 -0
  19. data/lib/fluent_plugin_kinesis/version.rb +10 -10
  20. metadata +18 -70
  21. data/README-v0.4.md +0 -348
  22. data/benchmark/dummy.conf +0 -0
  23. data/gemfiles/Gemfile.aws-sdk-2.4 +0 -20
  24. data/gemfiles/Gemfile.fluentd-0.10.58 +0 -20
  25. data/gemfiles/Gemfile.fluentd-0.14.11 +0 -20
  26. data/gemfiles/Gemfile.ruby-2.0 +0 -21
  27. data/gemfiles/Gemfile.ruby-2.1 +0 -21
  28. data/lib/fluent/plugin/kinesis_helper.rb +0 -36
  29. data/lib/fluent/plugin/kinesis_helper/class_methods.rb +0 -123
  30. data/lib/fluent/plugin/kinesis_helper/credentials.rb +0 -51
  31. data/lib/fluent/plugin/kinesis_helper/error.rb +0 -43
  32. data/lib/fluent/plugin/kinesis_helper/format.rb +0 -85
  33. data/lib/fluent/plugin/kinesis_helper/initialize.rb +0 -59
  34. data/lib/fluent/plugin/kinesis_helper/kpl.rb +0 -82
  35. data/lib/fluent/plugin/out_kinesis.rb +0 -323
  36. data/lib/fluent/plugin/out_kinesis_producer.rb +0 -48
  37. data/lib/fluent/plugin/patched_detach_process_impl.rb +0 -103
  38. data/lib/kinesis_producer.rb +0 -24
  39. data/lib/kinesis_producer/binary.rb +0 -10
  40. data/lib/kinesis_producer/daemon.rb +0 -270
  41. data/lib/kinesis_producer/library.rb +0 -122
  42. data/lib/kinesis_producer/protobuf/config.pb.rb +0 -66
  43. data/lib/kinesis_producer/protobuf/messages.pb.rb +0 -151
  44. data/lib/kinesis_producer/tasks/binary.rake +0 -73
@@ -1,151 +0,0 @@
1
- #
2
- # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- #
4
- # Licensed under the Amazon Software License (the "License").
5
- # You may not use this file except in compliance with the License.
6
- # A copy of the License is located at
7
- #
8
- # http://aws.amazon.com/asl/
9
- #
10
- # or in the "license" file accompanying this file. This file is distributed
11
- # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
- # express or implied. See the License for the specific language governing
13
- # permissions and limitations under the License.
14
-
15
- ##
16
- # This file is auto-generated. DO NOT EDIT!
17
- #
18
- require 'protobuf/message'
19
-
20
-
21
- ##
22
- # Imports
23
- #
24
- require 'kinesis_producer/protobuf/config.pb'
25
-
26
- module KinesisProducer
27
- module Protobuf
28
-
29
- ##
30
- # Message Classes
31
- #
32
- class Tag < ::Protobuf::Message; end
33
- class Record < ::Protobuf::Message; end
34
- class AggregatedRecord < ::Protobuf::Message; end
35
- class Message < ::Protobuf::Message; end
36
- class PutRecord < ::Protobuf::Message; end
37
- class Flush < ::Protobuf::Message; end
38
- class Attempt < ::Protobuf::Message; end
39
- class PutRecordResult < ::Protobuf::Message; end
40
- class Credentials < ::Protobuf::Message; end
41
- class SetCredentials < ::Protobuf::Message; end
42
- class Dimension < ::Protobuf::Message; end
43
- class Stats < ::Protobuf::Message; end
44
- class Metric < ::Protobuf::Message; end
45
- class MetricsRequest < ::Protobuf::Message; end
46
- class MetricsResponse < ::Protobuf::Message; end
47
-
48
-
49
- ##
50
- # Message Fields
51
- #
52
- class Tag
53
- required :string, :key, 1
54
- optional :string, :value, 2
55
- end
56
-
57
- class Record
58
- required :uint64, :partition_key_index, 1
59
- optional :uint64, :explicit_hash_key_index, 2
60
- required :bytes, :data, 3
61
- repeated ::KinesisProducer::Protobuf::Tag, :tags, 4
62
- end
63
-
64
- class AggregatedRecord
65
- repeated :string, :partition_key_table, 1
66
- repeated :string, :explicit_hash_key_table, 2
67
- repeated ::KinesisProducer::Protobuf::Record, :records, 3
68
- end
69
-
70
- class Message
71
- required :uint64, :id, 1
72
- optional :uint64, :source_id, 2
73
- optional ::KinesisProducer::Protobuf::PutRecord, :put_record, 3
74
- optional ::KinesisProducer::Protobuf::Flush, :flush, 4
75
- optional ::KinesisProducer::Protobuf::PutRecordResult, :put_record_result, 5
76
- optional ::KinesisProducer::Protobuf::Configuration, :configuration, 6
77
- optional ::KinesisProducer::Protobuf::MetricsRequest, :metrics_request, 7
78
- optional ::KinesisProducer::Protobuf::MetricsResponse, :metrics_response, 8
79
- optional ::KinesisProducer::Protobuf::SetCredentials, :set_credentials, 9
80
- end
81
-
82
- class PutRecord
83
- required :string, :stream_name, 1
84
- required :string, :partition_key, 2
85
- optional :string, :explicit_hash_key, 3
86
- required :bytes, :data, 4
87
- end
88
-
89
- class Flush
90
- optional :string, :stream_name, 1
91
- end
92
-
93
- class Attempt
94
- required :uint32, :delay, 1
95
- required :uint32, :duration, 2
96
- required :bool, :success, 3
97
- optional :string, :error_code, 4
98
- optional :string, :error_message, 5
99
- end
100
-
101
- class PutRecordResult
102
- repeated ::KinesisProducer::Protobuf::Attempt, :attempts, 1
103
- required :bool, :success, 2
104
- optional :string, :shard_id, 3
105
- optional :string, :sequence_number, 4
106
- end
107
-
108
- class Credentials
109
- required :string, :akid, 1
110
- required :string, :secret_key, 2
111
- optional :string, :token, 3
112
- end
113
-
114
- class SetCredentials
115
- optional :bool, :for_metrics, 1
116
- required ::KinesisProducer::Protobuf::Credentials, :credentials, 2
117
- end
118
-
119
- class Dimension
120
- required :string, :key, 1
121
- required :string, :value, 2
122
- end
123
-
124
- class Stats
125
- required :double, :count, 1
126
- required :double, :sum, 2
127
- required :double, :mean, 3
128
- required :double, :min, 4
129
- required :double, :max, 5
130
- end
131
-
132
- class Metric
133
- required :string, :name, 1
134
- repeated ::KinesisProducer::Protobuf::Dimension, :dimensions, 2
135
- required ::KinesisProducer::Protobuf::Stats, :stats, 3
136
- required :uint64, :seconds, 4
137
- end
138
-
139
- class MetricsRequest
140
- optional :string, :name, 1
141
- optional :uint64, :seconds, 2
142
- end
143
-
144
- class MetricsResponse
145
- repeated ::KinesisProducer::Protobuf::Metric, :metrics, 1
146
- end
147
-
148
- end
149
-
150
- end
151
-
@@ -1,73 +0,0 @@
1
- #
2
- # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- #
4
- # Licensed under the Amazon Software License (the "License").
5
- # You may not use this file except in compliance with the License.
6
- # A copy of the License is located at
7
- #
8
- # http://aws.amazon.com/asl/
9
- #
10
- # or in the "license" file accompanying this file. This file is distributed
11
- # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
- # express or implied. See the License for the specific language governing
13
- # permissions and limitations under the License.
14
-
15
- require 'kinesis_producer'
16
- require 'net/http'
17
- require 'zip'
18
-
19
- jar_version = "0.10.2"
20
- jar_file = "amazon-kinesis-producer-#{jar_version}.jar"
21
- jar_url = "https://search.maven.org/remotecontent?filepath=com/amazonaws/amazon-kinesis-producer/#{jar_version}/#{jar_file}"
22
- cache_dir = Pathname.new(".cache")
23
- cache_jar_file = cache_dir.join(jar_file)
24
- binaries = KinesisProducer::Binary::Files.values
25
-
26
- directory cache_dir
27
-
28
- file cache_jar_file => [cache_dir] do |t|
29
- puts "Downloading #{jar_file}"
30
- download(jar_url, t.name)
31
- end
32
-
33
- binaries.each do |bin|
34
- file bin => [cache_jar_file] do |t|
35
- puts "Extracting #{bin} from #{jar_file}"
36
- unzip(cache_jar_file, bin)
37
- chmod 0755, bin
38
- end
39
- end
40
-
41
- task :binaries => binaries
42
-
43
- task :clean do
44
- rm_rf cache_dir
45
- rm_rf File.dirname(File.dirname(binary))
46
- end
47
-
48
- def download(url, target)
49
- rm_f target
50
- uri = URI(url)
51
- Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
52
- req = Net::HTTP::Get.new(uri.request_uri)
53
- https.request(req) do |res|
54
- open(target, 'wb') do |io|
55
- res.read_body do |chunk|
56
- io.write(chunk)
57
- end
58
- end
59
- end
60
- end
61
- end
62
-
63
- def unzip(zip_file, *targets)
64
- rm_f targets
65
- Zip::File.open(zip_file) do |z|
66
- z.each do |entry|
67
- if targets.include?(entry.name)
68
- mkdir_p File.dirname(entry.name)
69
- entry.extract(entry.name)
70
- end
71
- end
72
- end
73
- end