fluent-plugin-timestream 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/README.md +13 -6
- data/fluent.conf.sample +36 -1
- data/lib/fluent/plugin/out_timestream.rb +77 -9
- data/lib/fluent/plugin/timestream/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c6da936981868da70ec9cd3817cc0b386c23b09723bb6b8ec8ccc5f2a8a0140f
|
4
|
+
data.tar.gz: d996e437118d2db8f58ed1e80a5a82412c8f08c666dcf827532bbc40ab588454
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d9855ee10803e4bcc1e451182e7ab650df68a7c5eaf62a273f5b8107a824ec23831f98746399ffddf525465e673ba522579a37f167ae16847cb7c830ea06d129
|
7
|
+
data.tar.gz: 4c93d1dd0ebe5c46bcf801e4e88a03da873272a1d70ceff454ae1a39f7d2c0d730777a50470e59ff93546d05e06cde298ca07cee5dcdeb840553719610c45bed
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -1,21 +1,28 @@
|
|
1
1
|
# fluent-plugin-timestream
|
2
|
+
|
2
3
|
[![Status](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream.svg?style=shield)](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream)
|
3
4
|
|
4
5
|
Fluentd output plugin for Amazon Timestream.
|
5
6
|
|
6
|
-
|
7
7
|
## Installation
|
8
|
+
|
8
9
|
You can install it as follows:
|
9
10
|
|
10
11
|
$ fluent-gem install fluent-plugin-timestream
|
11
12
|
|
12
13
|
## Configuration
|
14
|
+
|
13
15
|
Please refer to the [sample config file](https://github.com/StudistCorporation/fluent-plugin-timestream/blob/main/fluent.conf.sample)
|
14
16
|
|
15
17
|
## Note
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
|
19
|
+
The plugin ignores dimension when it has `null` or empty string value.
|
20
|
+
e.g. `{dimension1: null, dimension2: "value", dimension3: ""}` => `{dimension2: "value"}`
|
21
|
+
|
22
|
+
The plugin ignores record when it has no dimensions.
|
23
|
+
e.g. `{dimension1: null, dimension2: "", measure: "value"}` => ignores this record
|
24
|
+
|
25
|
+
The plugin ignores record when measure specified in the config has `null` or empty value.
|
26
|
+
e.g. `{dimension1: "value", measure: ""}` => ignores this record
|
27
|
+
|
21
28
|
Configuring multiple `MeasureName`s is not supported.
|
data/fluent.conf.sample
CHANGED
@@ -1,11 +1,35 @@
|
|
1
1
|
<source>
|
2
2
|
@type tail
|
3
|
-
format
|
3
|
+
format json
|
4
4
|
path /path/to/sample.log
|
5
5
|
pos_file /tmp/sample.pos
|
6
6
|
tag "sample.log"
|
7
|
+
# set keep_time_key true if need to convert time.
|
8
|
+
# keep_time_key true
|
7
9
|
</source>
|
8
10
|
|
11
|
+
# If more than seconds accuracy is needed,
|
12
|
+
# convert time and configure time_unit and time_key.
|
13
|
+
#
|
14
|
+
# e.g.)
|
15
|
+
# If record is as follows:
|
16
|
+
# {"key": "value", "time":1620000000.123456789}
|
17
|
+
#
|
18
|
+
# convert it to:
|
19
|
+
# when time unit is MILLISECONDS: 1620000000123
|
20
|
+
# when time unit is MICROSECONDS: 1620000000123456
|
21
|
+
# when time unit is NANOSECONDS: 1620000000123456789
|
22
|
+
# <filter sample.log>
|
23
|
+
# @type record_transformer
|
24
|
+
# enable_ruby true
|
25
|
+
# <record>
|
26
|
+
# milliseconds_time ${(record["time"].to_f * 1000).to_i}
|
27
|
+
# </record>
|
28
|
+
# Remove time key or plugin sends it as a dimension.
|
29
|
+
# Especially, if time key name is 'time', the plugin fails to write record because 'time' is reserved keyword.
|
30
|
+
# remove_keys time
|
31
|
+
#</filter>
|
32
|
+
|
9
33
|
<match sample.log>
|
10
34
|
@type timestream
|
11
35
|
|
@@ -24,6 +48,17 @@
|
|
24
48
|
aws_sec_key "XXXXXXXX"
|
25
49
|
region "us-east-1"
|
26
50
|
|
51
|
+
# If more than seconds accuracy is needed,
|
52
|
+
# convert time and configure time_unit and time_key.
|
53
|
+
# time_unit: The granularity of the timestamp unit.
|
54
|
+
# This value is used for 'TimeUnit' in Timestream record
|
55
|
+
# Default value: SECONDS
|
56
|
+
# Valid values: SECONDS MILLISECONDS MICROSECONDS NANOSECONDS
|
57
|
+
# time_key: Specify record key which contains integer epoch time corresponding to time_unit
|
58
|
+
# Plugin uses specified epoch time for 'Time' in Timestream record and does not send it as dimension
|
59
|
+
# time_unit "MILLISECONDS"
|
60
|
+
# time_key "milliseconds_time"
|
61
|
+
|
27
62
|
# Specify which key should be 'measure'.
|
28
63
|
# If not, plugin sends dummy measure and writes all keys and values as dimensions.
|
29
64
|
# Dummy measure is as follows:
|
@@ -6,8 +6,32 @@ require_relative 'timestream/version'
|
|
6
6
|
|
7
7
|
module Fluent
|
8
8
|
module Plugin
|
9
|
+
# rubocop: disable Metrics/ClassLength
|
9
10
|
# Fluent plugin for Amazon Timestream
|
10
11
|
class TimestreamOutput < Fluent::Plugin::Output
|
12
|
+
|
13
|
+
VALID_TIME_UNIT =
|
14
|
+
%w[
|
15
|
+
SECONDS
|
16
|
+
MILLISECONDS
|
17
|
+
MICROSECONDS
|
18
|
+
NANOSECONDS
|
19
|
+
].freeze
|
20
|
+
|
21
|
+
# Raise when measure has empty value
|
22
|
+
class EmptyValueError < StandardError
|
23
|
+
def initialize(key_name = '')
|
24
|
+
super("measure has empty value. key name: #{key_name}")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# Raise when record has no dimensions
|
29
|
+
class NoDimensionsError < StandardError
|
30
|
+
def initialize
|
31
|
+
super('record has no dimensions.')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
11
35
|
Fluent::Plugin.register_output('timestream', self)
|
12
36
|
|
13
37
|
config_param :region, :string, default: nil
|
@@ -22,6 +46,8 @@ module Fluent
|
|
22
46
|
config_param :name, :string
|
23
47
|
config_param :type, :string
|
24
48
|
end
|
49
|
+
config_param :time_unit, :string, default: 'SECONDS'
|
50
|
+
config_param :time_key, default: nil
|
25
51
|
|
26
52
|
config_param :endpoint, :string, default: nil
|
27
53
|
config_param :ssl_verify_peer, :bool, default: true
|
@@ -36,6 +62,7 @@ module Fluent
|
|
36
62
|
|
37
63
|
@database = ENV['AWS_TIMESTREAM_DATABASE'] if @database.nil?
|
38
64
|
@table = ENV['AWS_TIMESTREAM_TABLE'] if @table.nil?
|
65
|
+
validate_time_unit
|
39
66
|
end
|
40
67
|
|
41
68
|
def credential_options
|
@@ -49,6 +76,11 @@ module Fluent
|
|
49
76
|
end
|
50
77
|
end
|
51
78
|
|
79
|
+
def validate_time_unit
|
80
|
+
return if VALID_TIME_UNIT.include?(@time_unit)
|
81
|
+
raise Fluent::ConfigError, "Invalid time_unit: #{@time_unit}"
|
82
|
+
end
|
83
|
+
|
52
84
|
def formatted_to_msgpack_binary
|
53
85
|
true
|
54
86
|
end
|
@@ -58,60 +90,96 @@ module Fluent
|
|
58
90
|
end
|
59
91
|
|
60
92
|
def create_timestream_record(dimensions, time, measure)
|
93
|
+
raise NoDimensionsError if dimensions.empty?
|
61
94
|
measure = { name: '-', value: '-', type: 'VARCHAR' } if measure.empty?
|
62
95
|
{
|
63
96
|
dimensions: dimensions,
|
64
97
|
time: time.to_s,
|
65
|
-
time_unit:
|
98
|
+
time_unit: @time_unit,
|
66
99
|
measure_name: measure[:name],
|
67
|
-
measure_value: measure[:value]
|
100
|
+
measure_value: measure[:value],
|
68
101
|
measure_value_type: measure[:type]
|
69
102
|
}
|
70
103
|
end
|
71
104
|
|
72
105
|
def create_timestream_dimension(key, value)
|
106
|
+
value = value.to_s
|
107
|
+
|
108
|
+
# Timestream does not accept empty string.
|
109
|
+
# Ignore this dimension.
|
110
|
+
return nil if value.empty?
|
111
|
+
|
73
112
|
{
|
74
113
|
dimension_value_type: 'VARCHAR',
|
75
114
|
name: key,
|
76
|
-
value: value
|
115
|
+
value: value
|
116
|
+
}
|
117
|
+
end
|
118
|
+
|
119
|
+
def create_timestream_measure(key, value)
|
120
|
+
value = value.to_s
|
121
|
+
|
122
|
+
# Timestream does not accept empty string.
|
123
|
+
# By raising error, ignore entire record.
|
124
|
+
raise EmptyValueError, key if value.empty?
|
125
|
+
|
126
|
+
{
|
127
|
+
name: key,
|
128
|
+
value: value,
|
129
|
+
type: @target_measure[:type]
|
77
130
|
}
|
78
131
|
end
|
79
132
|
|
80
133
|
def create_timestream_dimensions_and_measure(record)
|
81
|
-
dimensions = []
|
82
134
|
measure = {}
|
83
|
-
record.
|
135
|
+
dimensions = record.each_with_object([]) do |(k, v), result|
|
84
136
|
if @target_measure && k == @target_measure[:name]
|
85
|
-
measure =
|
137
|
+
measure = create_timestream_measure(k, v)
|
86
138
|
next
|
87
139
|
end
|
88
|
-
|
140
|
+
dimension = create_timestream_dimension(k, v)
|
141
|
+
result.push(dimension) unless dimension.nil?
|
89
142
|
end
|
90
143
|
return [dimensions, measure]
|
91
144
|
end
|
92
145
|
|
146
|
+
# rubocop:disable Metrics/MethodLength
|
93
147
|
def create_timestream_records(chunk)
|
94
148
|
timestream_records = []
|
95
149
|
chunk.each do |time, record|
|
150
|
+
time = record.delete(@time_key) unless @time_key.nil?
|
96
151
|
dimensions, measure = create_timestream_dimensions_and_measure(record)
|
97
152
|
timestream_records.push(create_timestream_record(dimensions, time, measure))
|
153
|
+
rescue EmptyValueError, NoDimensionsError => e
|
154
|
+
log.warn("ignored record due to (#{e})")
|
155
|
+
log.debug("ignored record details: #{record}")
|
156
|
+
next
|
98
157
|
end
|
99
158
|
|
100
|
-
log.info("read #{timestream_records.length} records from chunk")
|
101
159
|
timestream_records
|
102
160
|
end
|
161
|
+
# rubocop:enable Metrics/MethodLength
|
103
162
|
|
104
163
|
def write(chunk)
|
164
|
+
records = create_timestream_records(chunk)
|
165
|
+
log.info("read #{records.length} records from chunk")
|
166
|
+
write_records(records)
|
167
|
+
end
|
168
|
+
|
169
|
+
def write_records(records)
|
170
|
+
return if records.empty?
|
105
171
|
@client.write_records(
|
106
172
|
database_name: @database,
|
107
173
|
table_name: @table,
|
108
|
-
records:
|
174
|
+
records: records
|
109
175
|
)
|
110
176
|
rescue Aws::TimestreamWrite::Errors::RejectedRecordsException => e
|
111
177
|
log.error(e.rejected_records)
|
112
178
|
rescue StandardError => e
|
113
179
|
log.error(e.message)
|
114
180
|
end
|
181
|
+
|
115
182
|
end
|
183
|
+
# rubocop: enable Metrics/ClassLength
|
116
184
|
end
|
117
185
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-timestream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Studist Corporation
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-05-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk-timestreamwrite
|