fluent-plugin-kinesis 1.3.0 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +2 -23
- data/CHANGELOG.md +13 -0
- data/Gemfile +9 -9
- data/LICENSE.txt +201 -40
- data/Makefile +24 -31
- data/README.md +179 -308
- data/Rakefile +9 -13
- data/benchmark/task.rake +96 -58
- data/fluent-plugin-kinesis.gemspec +15 -19
- data/gemfiles/Gemfile.fluentd-0.12 +10 -10
- data/lib/fluent/plugin/kinesis.rb +166 -0
- data/lib/fluent/plugin/kinesis_helper/aggregator.rb +99 -0
- data/lib/fluent/plugin/kinesis_helper/api.rb +152 -121
- data/lib/fluent/plugin/kinesis_helper/client.rb +125 -12
- data/lib/fluent/plugin/out_kinesis_firehose.rb +40 -27
- data/lib/fluent/plugin/out_kinesis_streams.rb +51 -30
- data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +76 -0
- data/lib/fluent_plugin_kinesis/version.rb +10 -10
- metadata +18 -70
- data/README-v0.4.md +0 -348
- data/benchmark/dummy.conf +0 -0
- data/gemfiles/Gemfile.aws-sdk-2.4 +0 -20
- data/gemfiles/Gemfile.fluentd-0.10.58 +0 -20
- data/gemfiles/Gemfile.fluentd-0.14.11 +0 -20
- data/gemfiles/Gemfile.ruby-2.0 +0 -21
- data/gemfiles/Gemfile.ruby-2.1 +0 -21
- data/lib/fluent/plugin/kinesis_helper.rb +0 -36
- data/lib/fluent/plugin/kinesis_helper/class_methods.rb +0 -123
- data/lib/fluent/plugin/kinesis_helper/credentials.rb +0 -51
- data/lib/fluent/plugin/kinesis_helper/error.rb +0 -43
- data/lib/fluent/plugin/kinesis_helper/format.rb +0 -85
- data/lib/fluent/plugin/kinesis_helper/initialize.rb +0 -59
- data/lib/fluent/plugin/kinesis_helper/kpl.rb +0 -82
- data/lib/fluent/plugin/out_kinesis.rb +0 -323
- data/lib/fluent/plugin/out_kinesis_producer.rb +0 -48
- data/lib/fluent/plugin/patched_detach_process_impl.rb +0 -103
- data/lib/kinesis_producer.rb +0 -24
- data/lib/kinesis_producer/binary.rb +0 -10
- data/lib/kinesis_producer/daemon.rb +0 -270
- data/lib/kinesis_producer/library.rb +0 -122
- data/lib/kinesis_producer/protobuf/config.pb.rb +0 -66
- data/lib/kinesis_producer/protobuf/messages.pb.rb +0 -151
- data/lib/kinesis_producer/tasks/binary.rake +0 -73
@@ -1,151 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
-
#
|
4
|
-
# Licensed under the Amazon Software License (the "License").
|
5
|
-
# You may not use this file except in compliance with the License.
|
6
|
-
# A copy of the License is located at
|
7
|
-
#
|
8
|
-
# http://aws.amazon.com/asl/
|
9
|
-
#
|
10
|
-
# or in the "license" file accompanying this file. This file is distributed
|
11
|
-
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
-
# express or implied. See the License for the specific language governing
|
13
|
-
# permissions and limitations under the License.
|
14
|
-
|
15
|
-
##
|
16
|
-
# This file is auto-generated. DO NOT EDIT!
|
17
|
-
#
|
18
|
-
require 'protobuf/message'
|
19
|
-
|
20
|
-
|
21
|
-
##
|
22
|
-
# Imports
|
23
|
-
#
|
24
|
-
require 'kinesis_producer/protobuf/config.pb'
|
25
|
-
|
26
|
-
module KinesisProducer
|
27
|
-
module Protobuf
|
28
|
-
|
29
|
-
##
|
30
|
-
# Message Classes
|
31
|
-
#
|
32
|
-
class Tag < ::Protobuf::Message; end
|
33
|
-
class Record < ::Protobuf::Message; end
|
34
|
-
class AggregatedRecord < ::Protobuf::Message; end
|
35
|
-
class Message < ::Protobuf::Message; end
|
36
|
-
class PutRecord < ::Protobuf::Message; end
|
37
|
-
class Flush < ::Protobuf::Message; end
|
38
|
-
class Attempt < ::Protobuf::Message; end
|
39
|
-
class PutRecordResult < ::Protobuf::Message; end
|
40
|
-
class Credentials < ::Protobuf::Message; end
|
41
|
-
class SetCredentials < ::Protobuf::Message; end
|
42
|
-
class Dimension < ::Protobuf::Message; end
|
43
|
-
class Stats < ::Protobuf::Message; end
|
44
|
-
class Metric < ::Protobuf::Message; end
|
45
|
-
class MetricsRequest < ::Protobuf::Message; end
|
46
|
-
class MetricsResponse < ::Protobuf::Message; end
|
47
|
-
|
48
|
-
|
49
|
-
##
|
50
|
-
# Message Fields
|
51
|
-
#
|
52
|
-
class Tag
|
53
|
-
required :string, :key, 1
|
54
|
-
optional :string, :value, 2
|
55
|
-
end
|
56
|
-
|
57
|
-
class Record
|
58
|
-
required :uint64, :partition_key_index, 1
|
59
|
-
optional :uint64, :explicit_hash_key_index, 2
|
60
|
-
required :bytes, :data, 3
|
61
|
-
repeated ::KinesisProducer::Protobuf::Tag, :tags, 4
|
62
|
-
end
|
63
|
-
|
64
|
-
class AggregatedRecord
|
65
|
-
repeated :string, :partition_key_table, 1
|
66
|
-
repeated :string, :explicit_hash_key_table, 2
|
67
|
-
repeated ::KinesisProducer::Protobuf::Record, :records, 3
|
68
|
-
end
|
69
|
-
|
70
|
-
class Message
|
71
|
-
required :uint64, :id, 1
|
72
|
-
optional :uint64, :source_id, 2
|
73
|
-
optional ::KinesisProducer::Protobuf::PutRecord, :put_record, 3
|
74
|
-
optional ::KinesisProducer::Protobuf::Flush, :flush, 4
|
75
|
-
optional ::KinesisProducer::Protobuf::PutRecordResult, :put_record_result, 5
|
76
|
-
optional ::KinesisProducer::Protobuf::Configuration, :configuration, 6
|
77
|
-
optional ::KinesisProducer::Protobuf::MetricsRequest, :metrics_request, 7
|
78
|
-
optional ::KinesisProducer::Protobuf::MetricsResponse, :metrics_response, 8
|
79
|
-
optional ::KinesisProducer::Protobuf::SetCredentials, :set_credentials, 9
|
80
|
-
end
|
81
|
-
|
82
|
-
class PutRecord
|
83
|
-
required :string, :stream_name, 1
|
84
|
-
required :string, :partition_key, 2
|
85
|
-
optional :string, :explicit_hash_key, 3
|
86
|
-
required :bytes, :data, 4
|
87
|
-
end
|
88
|
-
|
89
|
-
class Flush
|
90
|
-
optional :string, :stream_name, 1
|
91
|
-
end
|
92
|
-
|
93
|
-
class Attempt
|
94
|
-
required :uint32, :delay, 1
|
95
|
-
required :uint32, :duration, 2
|
96
|
-
required :bool, :success, 3
|
97
|
-
optional :string, :error_code, 4
|
98
|
-
optional :string, :error_message, 5
|
99
|
-
end
|
100
|
-
|
101
|
-
class PutRecordResult
|
102
|
-
repeated ::KinesisProducer::Protobuf::Attempt, :attempts, 1
|
103
|
-
required :bool, :success, 2
|
104
|
-
optional :string, :shard_id, 3
|
105
|
-
optional :string, :sequence_number, 4
|
106
|
-
end
|
107
|
-
|
108
|
-
class Credentials
|
109
|
-
required :string, :akid, 1
|
110
|
-
required :string, :secret_key, 2
|
111
|
-
optional :string, :token, 3
|
112
|
-
end
|
113
|
-
|
114
|
-
class SetCredentials
|
115
|
-
optional :bool, :for_metrics, 1
|
116
|
-
required ::KinesisProducer::Protobuf::Credentials, :credentials, 2
|
117
|
-
end
|
118
|
-
|
119
|
-
class Dimension
|
120
|
-
required :string, :key, 1
|
121
|
-
required :string, :value, 2
|
122
|
-
end
|
123
|
-
|
124
|
-
class Stats
|
125
|
-
required :double, :count, 1
|
126
|
-
required :double, :sum, 2
|
127
|
-
required :double, :mean, 3
|
128
|
-
required :double, :min, 4
|
129
|
-
required :double, :max, 5
|
130
|
-
end
|
131
|
-
|
132
|
-
class Metric
|
133
|
-
required :string, :name, 1
|
134
|
-
repeated ::KinesisProducer::Protobuf::Dimension, :dimensions, 2
|
135
|
-
required ::KinesisProducer::Protobuf::Stats, :stats, 3
|
136
|
-
required :uint64, :seconds, 4
|
137
|
-
end
|
138
|
-
|
139
|
-
class MetricsRequest
|
140
|
-
optional :string, :name, 1
|
141
|
-
optional :uint64, :seconds, 2
|
142
|
-
end
|
143
|
-
|
144
|
-
class MetricsResponse
|
145
|
-
repeated ::KinesisProducer::Protobuf::Metric, :metrics, 1
|
146
|
-
end
|
147
|
-
|
148
|
-
end
|
149
|
-
|
150
|
-
end
|
151
|
-
|
@@ -1,73 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
-
#
|
4
|
-
# Licensed under the Amazon Software License (the "License").
|
5
|
-
# You may not use this file except in compliance with the License.
|
6
|
-
# A copy of the License is located at
|
7
|
-
#
|
8
|
-
# http://aws.amazon.com/asl/
|
9
|
-
#
|
10
|
-
# or in the "license" file accompanying this file. This file is distributed
|
11
|
-
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
-
# express or implied. See the License for the specific language governing
|
13
|
-
# permissions and limitations under the License.
|
14
|
-
|
15
|
-
require 'kinesis_producer'
|
16
|
-
require 'net/http'
|
17
|
-
require 'zip'
|
18
|
-
|
19
|
-
jar_version = "0.10.2"
|
20
|
-
jar_file = "amazon-kinesis-producer-#{jar_version}.jar"
|
21
|
-
jar_url = "https://search.maven.org/remotecontent?filepath=com/amazonaws/amazon-kinesis-producer/#{jar_version}/#{jar_file}"
|
22
|
-
cache_dir = Pathname.new(".cache")
|
23
|
-
cache_jar_file = cache_dir.join(jar_file)
|
24
|
-
binaries = KinesisProducer::Binary::Files.values
|
25
|
-
|
26
|
-
directory cache_dir
|
27
|
-
|
28
|
-
file cache_jar_file => [cache_dir] do |t|
|
29
|
-
puts "Downloading #{jar_file}"
|
30
|
-
download(jar_url, t.name)
|
31
|
-
end
|
32
|
-
|
33
|
-
binaries.each do |bin|
|
34
|
-
file bin => [cache_jar_file] do |t|
|
35
|
-
puts "Extracting #{bin} from #{jar_file}"
|
36
|
-
unzip(cache_jar_file, bin)
|
37
|
-
chmod 0755, bin
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
task :binaries => binaries
|
42
|
-
|
43
|
-
task :clean do
|
44
|
-
rm_rf cache_dir
|
45
|
-
rm_rf File.dirname(File.dirname(binary))
|
46
|
-
end
|
47
|
-
|
48
|
-
def download(url, target)
|
49
|
-
rm_f target
|
50
|
-
uri = URI(url)
|
51
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
|
52
|
-
req = Net::HTTP::Get.new(uri.request_uri)
|
53
|
-
https.request(req) do |res|
|
54
|
-
open(target, 'wb') do |io|
|
55
|
-
res.read_body do |chunk|
|
56
|
-
io.write(chunk)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def unzip(zip_file, *targets)
|
64
|
-
rm_f targets
|
65
|
-
Zip::File.open(zip_file) do |z|
|
66
|
-
z.each do |entry|
|
67
|
-
if targets.include?(entry.name)
|
68
|
-
mkdir_p File.dirname(entry.name)
|
69
|
-
entry.extract(entry.name)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|