fluent-plugin-kinesis 1.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -23
- data/CHANGELOG.md +13 -0
- data/Gemfile +9 -9
- data/LICENSE.txt +201 -40
- data/Makefile +24 -31
- data/README.md +179 -308
- data/Rakefile +9 -13
- data/benchmark/task.rake +96 -58
- data/fluent-plugin-kinesis.gemspec +15 -19
- data/gemfiles/Gemfile.fluentd-0.12 +10 -10
- data/lib/fluent/plugin/kinesis.rb +166 -0
- data/lib/fluent/plugin/kinesis_helper/aggregator.rb +99 -0
- data/lib/fluent/plugin/kinesis_helper/api.rb +152 -121
- data/lib/fluent/plugin/kinesis_helper/client.rb +125 -12
- data/lib/fluent/plugin/out_kinesis_firehose.rb +40 -27
- data/lib/fluent/plugin/out_kinesis_streams.rb +51 -30
- data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +76 -0
- data/lib/fluent_plugin_kinesis/version.rb +10 -10
- metadata +18 -70
- data/README-v0.4.md +0 -348
- data/benchmark/dummy.conf +0 -0
- data/gemfiles/Gemfile.aws-sdk-2.4 +0 -20
- data/gemfiles/Gemfile.fluentd-0.10.58 +0 -20
- data/gemfiles/Gemfile.fluentd-0.14.11 +0 -20
- data/gemfiles/Gemfile.ruby-2.0 +0 -21
- data/gemfiles/Gemfile.ruby-2.1 +0 -21
- data/lib/fluent/plugin/kinesis_helper.rb +0 -36
- data/lib/fluent/plugin/kinesis_helper/class_methods.rb +0 -123
- data/lib/fluent/plugin/kinesis_helper/credentials.rb +0 -51
- data/lib/fluent/plugin/kinesis_helper/error.rb +0 -43
- data/lib/fluent/plugin/kinesis_helper/format.rb +0 -85
- data/lib/fluent/plugin/kinesis_helper/initialize.rb +0 -59
- data/lib/fluent/plugin/kinesis_helper/kpl.rb +0 -82
- data/lib/fluent/plugin/out_kinesis.rb +0 -323
- data/lib/fluent/plugin/out_kinesis_producer.rb +0 -48
- data/lib/fluent/plugin/patched_detach_process_impl.rb +0 -103
- data/lib/kinesis_producer.rb +0 -24
- data/lib/kinesis_producer/binary.rb +0 -10
- data/lib/kinesis_producer/daemon.rb +0 -270
- data/lib/kinesis_producer/library.rb +0 -122
- data/lib/kinesis_producer/protobuf/config.pb.rb +0 -66
- data/lib/kinesis_producer/protobuf/messages.pb.rb +0 -151
- data/lib/kinesis_producer/tasks/binary.rake +0 -73
@@ -1,151 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
-
#
|
4
|
-
# Licensed under the Amazon Software License (the "License").
|
5
|
-
# You may not use this file except in compliance with the License.
|
6
|
-
# A copy of the License is located at
|
7
|
-
#
|
8
|
-
# http://aws.amazon.com/asl/
|
9
|
-
#
|
10
|
-
# or in the "license" file accompanying this file. This file is distributed
|
11
|
-
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
-
# express or implied. See the License for the specific language governing
|
13
|
-
# permissions and limitations under the License.
|
14
|
-
|
15
|
-
##
|
16
|
-
# This file is auto-generated. DO NOT EDIT!
|
17
|
-
#
|
18
|
-
require 'protobuf/message'
|
19
|
-
|
20
|
-
|
21
|
-
##
|
22
|
-
# Imports
|
23
|
-
#
|
24
|
-
require 'kinesis_producer/protobuf/config.pb'
|
25
|
-
|
26
|
-
module KinesisProducer
|
27
|
-
module Protobuf
|
28
|
-
|
29
|
-
##
|
30
|
-
# Message Classes
|
31
|
-
#
|
32
|
-
class Tag < ::Protobuf::Message; end
|
33
|
-
class Record < ::Protobuf::Message; end
|
34
|
-
class AggregatedRecord < ::Protobuf::Message; end
|
35
|
-
class Message < ::Protobuf::Message; end
|
36
|
-
class PutRecord < ::Protobuf::Message; end
|
37
|
-
class Flush < ::Protobuf::Message; end
|
38
|
-
class Attempt < ::Protobuf::Message; end
|
39
|
-
class PutRecordResult < ::Protobuf::Message; end
|
40
|
-
class Credentials < ::Protobuf::Message; end
|
41
|
-
class SetCredentials < ::Protobuf::Message; end
|
42
|
-
class Dimension < ::Protobuf::Message; end
|
43
|
-
class Stats < ::Protobuf::Message; end
|
44
|
-
class Metric < ::Protobuf::Message; end
|
45
|
-
class MetricsRequest < ::Protobuf::Message; end
|
46
|
-
class MetricsResponse < ::Protobuf::Message; end
|
47
|
-
|
48
|
-
|
49
|
-
##
|
50
|
-
# Message Fields
|
51
|
-
#
|
52
|
-
class Tag
|
53
|
-
required :string, :key, 1
|
54
|
-
optional :string, :value, 2
|
55
|
-
end
|
56
|
-
|
57
|
-
class Record
|
58
|
-
required :uint64, :partition_key_index, 1
|
59
|
-
optional :uint64, :explicit_hash_key_index, 2
|
60
|
-
required :bytes, :data, 3
|
61
|
-
repeated ::KinesisProducer::Protobuf::Tag, :tags, 4
|
62
|
-
end
|
63
|
-
|
64
|
-
class AggregatedRecord
|
65
|
-
repeated :string, :partition_key_table, 1
|
66
|
-
repeated :string, :explicit_hash_key_table, 2
|
67
|
-
repeated ::KinesisProducer::Protobuf::Record, :records, 3
|
68
|
-
end
|
69
|
-
|
70
|
-
class Message
|
71
|
-
required :uint64, :id, 1
|
72
|
-
optional :uint64, :source_id, 2
|
73
|
-
optional ::KinesisProducer::Protobuf::PutRecord, :put_record, 3
|
74
|
-
optional ::KinesisProducer::Protobuf::Flush, :flush, 4
|
75
|
-
optional ::KinesisProducer::Protobuf::PutRecordResult, :put_record_result, 5
|
76
|
-
optional ::KinesisProducer::Protobuf::Configuration, :configuration, 6
|
77
|
-
optional ::KinesisProducer::Protobuf::MetricsRequest, :metrics_request, 7
|
78
|
-
optional ::KinesisProducer::Protobuf::MetricsResponse, :metrics_response, 8
|
79
|
-
optional ::KinesisProducer::Protobuf::SetCredentials, :set_credentials, 9
|
80
|
-
end
|
81
|
-
|
82
|
-
class PutRecord
|
83
|
-
required :string, :stream_name, 1
|
84
|
-
required :string, :partition_key, 2
|
85
|
-
optional :string, :explicit_hash_key, 3
|
86
|
-
required :bytes, :data, 4
|
87
|
-
end
|
88
|
-
|
89
|
-
class Flush
|
90
|
-
optional :string, :stream_name, 1
|
91
|
-
end
|
92
|
-
|
93
|
-
class Attempt
|
94
|
-
required :uint32, :delay, 1
|
95
|
-
required :uint32, :duration, 2
|
96
|
-
required :bool, :success, 3
|
97
|
-
optional :string, :error_code, 4
|
98
|
-
optional :string, :error_message, 5
|
99
|
-
end
|
100
|
-
|
101
|
-
class PutRecordResult
|
102
|
-
repeated ::KinesisProducer::Protobuf::Attempt, :attempts, 1
|
103
|
-
required :bool, :success, 2
|
104
|
-
optional :string, :shard_id, 3
|
105
|
-
optional :string, :sequence_number, 4
|
106
|
-
end
|
107
|
-
|
108
|
-
class Credentials
|
109
|
-
required :string, :akid, 1
|
110
|
-
required :string, :secret_key, 2
|
111
|
-
optional :string, :token, 3
|
112
|
-
end
|
113
|
-
|
114
|
-
class SetCredentials
|
115
|
-
optional :bool, :for_metrics, 1
|
116
|
-
required ::KinesisProducer::Protobuf::Credentials, :credentials, 2
|
117
|
-
end
|
118
|
-
|
119
|
-
class Dimension
|
120
|
-
required :string, :key, 1
|
121
|
-
required :string, :value, 2
|
122
|
-
end
|
123
|
-
|
124
|
-
class Stats
|
125
|
-
required :double, :count, 1
|
126
|
-
required :double, :sum, 2
|
127
|
-
required :double, :mean, 3
|
128
|
-
required :double, :min, 4
|
129
|
-
required :double, :max, 5
|
130
|
-
end
|
131
|
-
|
132
|
-
class Metric
|
133
|
-
required :string, :name, 1
|
134
|
-
repeated ::KinesisProducer::Protobuf::Dimension, :dimensions, 2
|
135
|
-
required ::KinesisProducer::Protobuf::Stats, :stats, 3
|
136
|
-
required :uint64, :seconds, 4
|
137
|
-
end
|
138
|
-
|
139
|
-
class MetricsRequest
|
140
|
-
optional :string, :name, 1
|
141
|
-
optional :uint64, :seconds, 2
|
142
|
-
end
|
143
|
-
|
144
|
-
class MetricsResponse
|
145
|
-
repeated ::KinesisProducer::Protobuf::Metric, :metrics, 1
|
146
|
-
end
|
147
|
-
|
148
|
-
end
|
149
|
-
|
150
|
-
end
|
151
|
-
|
@@ -1,73 +0,0 @@
|
|
1
|
-
#
|
2
|
-
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
-
#
|
4
|
-
# Licensed under the Amazon Software License (the "License").
|
5
|
-
# You may not use this file except in compliance with the License.
|
6
|
-
# A copy of the License is located at
|
7
|
-
#
|
8
|
-
# http://aws.amazon.com/asl/
|
9
|
-
#
|
10
|
-
# or in the "license" file accompanying this file. This file is distributed
|
11
|
-
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
-
# express or implied. See the License for the specific language governing
|
13
|
-
# permissions and limitations under the License.
|
14
|
-
|
15
|
-
require 'kinesis_producer'
|
16
|
-
require 'net/http'
|
17
|
-
require 'zip'
|
18
|
-
|
19
|
-
jar_version = "0.10.2"
|
20
|
-
jar_file = "amazon-kinesis-producer-#{jar_version}.jar"
|
21
|
-
jar_url = "https://search.maven.org/remotecontent?filepath=com/amazonaws/amazon-kinesis-producer/#{jar_version}/#{jar_file}"
|
22
|
-
cache_dir = Pathname.new(".cache")
|
23
|
-
cache_jar_file = cache_dir.join(jar_file)
|
24
|
-
binaries = KinesisProducer::Binary::Files.values
|
25
|
-
|
26
|
-
directory cache_dir
|
27
|
-
|
28
|
-
file cache_jar_file => [cache_dir] do |t|
|
29
|
-
puts "Downloading #{jar_file}"
|
30
|
-
download(jar_url, t.name)
|
31
|
-
end
|
32
|
-
|
33
|
-
binaries.each do |bin|
|
34
|
-
file bin => [cache_jar_file] do |t|
|
35
|
-
puts "Extracting #{bin} from #{jar_file}"
|
36
|
-
unzip(cache_jar_file, bin)
|
37
|
-
chmod 0755, bin
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
task :binaries => binaries
|
42
|
-
|
43
|
-
task :clean do
|
44
|
-
rm_rf cache_dir
|
45
|
-
rm_rf File.dirname(File.dirname(binary))
|
46
|
-
end
|
47
|
-
|
48
|
-
def download(url, target)
|
49
|
-
rm_f target
|
50
|
-
uri = URI(url)
|
51
|
-
Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |https|
|
52
|
-
req = Net::HTTP::Get.new(uri.request_uri)
|
53
|
-
https.request(req) do |res|
|
54
|
-
open(target, 'wb') do |io|
|
55
|
-
res.read_body do |chunk|
|
56
|
-
io.write(chunk)
|
57
|
-
end
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
def unzip(zip_file, *targets)
|
64
|
-
rm_f targets
|
65
|
-
Zip::File.open(zip_file) do |z|
|
66
|
-
z.each do |entry|
|
67
|
-
if targets.include?(entry.name)
|
68
|
-
mkdir_p File.dirname(entry.name)
|
69
|
-
entry.extract(entry.name)
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|