fluent-plugin-kinesis 1.3.0 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +2 -23
  3. data/CHANGELOG.md +13 -0
  4. data/Gemfile +9 -9
  5. data/LICENSE.txt +201 -40
  6. data/Makefile +24 -31
  7. data/README.md +179 -308
  8. data/Rakefile +9 -13
  9. data/benchmark/task.rake +96 -58
  10. data/fluent-plugin-kinesis.gemspec +15 -19
  11. data/gemfiles/Gemfile.fluentd-0.12 +10 -10
  12. data/lib/fluent/plugin/kinesis.rb +166 -0
  13. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +99 -0
  14. data/lib/fluent/plugin/kinesis_helper/api.rb +152 -121
  15. data/lib/fluent/plugin/kinesis_helper/client.rb +125 -12
  16. data/lib/fluent/plugin/out_kinesis_firehose.rb +40 -27
  17. data/lib/fluent/plugin/out_kinesis_streams.rb +51 -30
  18. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +76 -0
  19. data/lib/fluent_plugin_kinesis/version.rb +10 -10
  20. metadata +18 -70
  21. data/README-v0.4.md +0 -348
  22. data/benchmark/dummy.conf +0 -0
  23. data/gemfiles/Gemfile.aws-sdk-2.4 +0 -20
  24. data/gemfiles/Gemfile.fluentd-0.10.58 +0 -20
  25. data/gemfiles/Gemfile.fluentd-0.14.11 +0 -20
  26. data/gemfiles/Gemfile.ruby-2.0 +0 -21
  27. data/gemfiles/Gemfile.ruby-2.1 +0 -21
  28. data/lib/fluent/plugin/kinesis_helper.rb +0 -36
  29. data/lib/fluent/plugin/kinesis_helper/class_methods.rb +0 -123
  30. data/lib/fluent/plugin/kinesis_helper/credentials.rb +0 -51
  31. data/lib/fluent/plugin/kinesis_helper/error.rb +0 -43
  32. data/lib/fluent/plugin/kinesis_helper/format.rb +0 -85
  33. data/lib/fluent/plugin/kinesis_helper/initialize.rb +0 -59
  34. data/lib/fluent/plugin/kinesis_helper/kpl.rb +0 -82
  35. data/lib/fluent/plugin/out_kinesis.rb +0 -323
  36. data/lib/fluent/plugin/out_kinesis_producer.rb +0 -48
  37. data/lib/fluent/plugin/patched_detach_process_impl.rb +0 -103
  38. data/lib/kinesis_producer.rb +0 -24
  39. data/lib/kinesis_producer/binary.rb +0 -10
  40. data/lib/kinesis_producer/daemon.rb +0 -270
  41. data/lib/kinesis_producer/library.rb +0 -122
  42. data/lib/kinesis_producer/protobuf/config.pb.rb +0 -66
  43. data/lib/kinesis_producer/protobuf/messages.pb.rb +0 -151
  44. data/lib/kinesis_producer/tasks/binary.rake +0 -73
@@ -1,151 +0,0 @@
1
- #
2
- # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- #
4
- # Licensed under the Amazon Software License (the "License").
5
- # You may not use this file except in compliance with the License.
6
- # A copy of the License is located at
7
- #
8
- # http://aws.amazon.com/asl/
9
- #
10
- # or in the "license" file accompanying this file. This file is distributed
11
- # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
- # express or implied. See the License for the specific language governing
13
- # permissions and limitations under the License.
14
-
15
- ##
16
- # This file is auto-generated. DO NOT EDIT!
17
- #
18
- require 'protobuf/message'
19
-
20
-
21
- ##
22
- # Imports
23
- #
24
- require 'kinesis_producer/protobuf/config.pb'
25
-
26
- module KinesisProducer
27
- module Protobuf
28
-
29
- ##
30
- # Message Classes
31
- #
32
- class Tag < ::Protobuf::Message; end
33
- class Record < ::Protobuf::Message; end
34
- class AggregatedRecord < ::Protobuf::Message; end
35
- class Message < ::Protobuf::Message; end
36
- class PutRecord < ::Protobuf::Message; end
37
- class Flush < ::Protobuf::Message; end
38
- class Attempt < ::Protobuf::Message; end
39
- class PutRecordResult < ::Protobuf::Message; end
40
- class Credentials < ::Protobuf::Message; end
41
- class SetCredentials < ::Protobuf::Message; end
42
- class Dimension < ::Protobuf::Message; end
43
- class Stats < ::Protobuf::Message; end
44
- class Metric < ::Protobuf::Message; end
45
- class MetricsRequest < ::Protobuf::Message; end
46
- class MetricsResponse < ::Protobuf::Message; end
47
-
48
-
49
- ##
50
- # Message Fields
51
- #
52
- class Tag
53
- required :string, :key, 1
54
- optional :string, :value, 2
55
- end
56
-
57
- class Record
58
- required :uint64, :partition_key_index, 1
59
- optional :uint64, :explicit_hash_key_index, 2
60
- required :bytes, :data, 3
61
- repeated ::KinesisProducer::Protobuf::Tag, :tags, 4
62
- end
63
-
64
- class AggregatedRecord
65
- repeated :string, :partition_key_table, 1
66
- repeated :string, :explicit_hash_key_table, 2
67
- repeated ::KinesisProducer::Protobuf::Record, :records, 3
68
- end
69
-
70
- class Message
71
- required :uint64, :id, 1
72
- optional :uint64, :source_id, 2
73
- optional ::KinesisProducer::Protobuf::PutRecord, :put_record, 3
74
- optional ::KinesisProducer::Protobuf::Flush, :flush, 4
75
- optional ::KinesisProducer::Protobuf::PutRecordResult, :put_record_result, 5
76
- optional ::KinesisProducer::Protobuf::Configuration, :configuration, 6
77
- optional ::KinesisProducer::Protobuf::MetricsRequest, :metrics_request, 7
78
- optional ::KinesisProducer::Protobuf::MetricsResponse, :metrics_response, 8
79
- optional ::KinesisProducer::Protobuf::SetCredentials, :set_credentials, 9
80
- end
81
-
82
- class PutRecord
83
- required :string, :stream_name, 1
84
- required :string, :partition_key, 2
85
- optional :string, :explicit_hash_key, 3
86
- required :bytes, :data, 4
87
- end
88
-
89
- class Flush
90
- optional :string, :stream_name, 1
91
- end
92
-
93
- class Attempt
94
- required :uint32, :delay, 1
95
- required :uint32, :duration, 2
96
- required :bool, :success, 3
97
- optional :string, :error_code, 4
98
- optional :string, :error_message, 5
99
- end
100
-
101
- class PutRecordResult
102
- repeated ::KinesisProducer::Protobuf::Attempt, :attempts, 1
103
- required :bool, :success, 2
104
- optional :string, :shard_id, 3
105
- optional :string, :sequence_number, 4
106
- end
107
-
108
- class Credentials
109
- required :string, :akid, 1
110
- required :string, :secret_key, 2
111
- optional :string, :token, 3
112
- end
113
-
114
- class SetCredentials
115
- optional :bool, :for_metrics, 1
116
- required ::KinesisProducer::Protobuf::Credentials, :credentials, 2
117
- end
118
-
119
- class Dimension
120
- required :string, :key, 1
121
- required :string, :value, 2
122
- end
123
-
124
- class Stats
125
- required :double, :count, 1
126
- required :double, :sum, 2
127
- required :double, :mean, 3
128
- required :double, :min, 4
129
- required :double, :max, 5
130
- end
131
-
132
- class Metric
133
- required :string, :name, 1
134
- repeated ::KinesisProducer::Protobuf::Dimension, :dimensions, 2
135
- required ::KinesisProducer::Protobuf::Stats, :stats, 3
136
- required :uint64, :seconds, 4
137
- end
138
-
139
- class MetricsRequest
140
- optional :string, :name, 1
141
- optional :uint64, :seconds, 2
142
- end
143
-
144
- class MetricsResponse
145
- repeated ::KinesisProducer::Protobuf::Metric, :metrics, 1
146
- end
147
-
148
- end
149
-
150
- end
151
-
@@ -1,73 +0,0 @@
1
- #
2
- # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
- #
4
- # Licensed under the Amazon Software License (the "License").
5
- # You may not use this file except in compliance with the License.
6
- # A copy of the License is located at
7
- #
8
- # http://aws.amazon.com/asl/
9
- #
10
- # or in the "license" file accompanying this file. This file is distributed
11
- # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
- # express or implied. See the License for the specific language governing
13
- # permissions and limitations under the License.
14
-
15
- require 'kinesis_producer'
16
- require 'net/http'
17
- require 'zip'
18
-
19
- jar_version = "0.10.2"
20
- jar_file = "amazon-kinesis-producer-#{jar_version}.jar"
21
- jar_url = "https://search.maven.org/remotecontent?filepath=com/amazonaws/amazon-kinesis-producer/#{jar_version}/#{jar_file}"
22
- cache_dir = Pathname.new(".cache")
23
- cache_jar_file = cache_dir.join(jar_file)
24
- binaries = KinesisProducer::Binary::Files.values
25
-
26
- directory cache_dir
27
-
28
- file cache_jar_file => [cache_dir] do |t|
29
- puts "Downloading #{jar_file}"
30
- download(jar_url, t.name)
31
- end
32
-
33
- binaries.each do |bin|
34
- file bin => [cache_jar_file] do |t|
35
- puts "Extracting #{bin} from #{jar_file}"
36
- unzip(cache_jar_file, bin)
37
- chmod 0755, bin
38
- end
39
- end
40
-
41
- task :binaries => binaries
42
-
43
- task :clean do
44
- rm_rf cache_dir
45
- rm_rf File.dirname(File.dirname(binary))
46
- end
47
-
48
- def download(url, target)
49
- rm_f target
50
- uri = URI(url)
51
- Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
52
- req = Net::HTTP::Get.new(uri.request_uri)
53
- https.request(req) do |res|
54
- open(target, 'wb') do |io|
55
- res.read_body do |chunk|
56
- io.write(chunk)
57
- end
58
- end
59
- end
60
- end
61
- end
62
-
63
- def unzip(zip_file, *targets)
64
- rm_f targets
65
- Zip::File.open(zip_file) do |z|
66
- z.each do |entry|
67
- if targets.include?(entry.name)
68
- mkdir_p File.dirname(entry.name)
69
- entry.extract(entry.name)
70
- end
71
- end
72
- end
73
- end