fluent-plugin-output-solr 0.4.9 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +19 -3
- data/fluent-plugin-output-solr.gemspec +1 -1
- data/lib/fluent/plugin/out_solr.rb +25 -9
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb2ceab3cbbcea52fc45b798fcafc0101e0d0477
|
4
|
+
data.tar.gz: e68164616412a7a22e2dfd0a7c480b36cfbb9e33
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 88eedfd9043ffc95485489c43e96a7bbda53149c56f62ba515b40fe66edc1fda5765cfa98f4466c50bc5316908c8cf133a6ba706d314a8fe5a631a662094d4d4
|
7
|
+
data.tar.gz: be00852de009eacfc458478a5a25c313ddf318750e050925136fac4325ecdde1a90d017cc806087ad98e77aa92b6269d67464d0668948401c71929c60a6ca1dd
|
data/README.md
CHANGED
@@ -88,12 +88,28 @@ A string field value max length. If set -1, it means unlimited (default -1). How
|
|
88
88
|
string_field_value_max_length -1
|
89
89
|
```
|
90
90
|
|
91
|
-
###
|
91
|
+
### time_field
|
92
92
|
|
93
|
-
A field name of event timestamp in the Solr schema.xml (default
|
93
|
+
A field name of event timestamp in the Solr schema.xml (default time).
|
94
94
|
|
95
95
|
```
|
96
|
-
|
96
|
+
time_field time
|
97
|
+
```
|
98
|
+
|
99
|
+
### time_format
|
100
|
+
|
101
|
+
The format of the time field (default %FT%TZ).
|
102
|
+
|
103
|
+
```
|
104
|
+
time_format %FT%TZ
|
105
|
+
```
|
106
|
+
|
107
|
+
### time_output_format
|
108
|
+
|
109
|
+
The format of the time field of Solr. (default %FT%TZ).
|
110
|
+
|
111
|
+
```
|
112
|
+
time_output_format %FT%TZ
|
97
113
|
```
|
98
114
|
|
99
115
|
### flush_size
|
@@ -11,7 +11,9 @@ module Fluent
|
|
11
11
|
DEFAULT_IGNORE_UNDEFINED_FIELDS = false
|
12
12
|
DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
|
13
13
|
DEFAULT_TAG_FIELD = 'tag'
|
14
|
-
|
14
|
+
DEFAULT_TIME_FIELD = 'time'
|
15
|
+
DEFAULT_TIME_FORMAT = '%FT%TZ'
|
16
|
+
DEFAULT_TIME_OUTPUT_FORMAT = '%FT%TZ'
|
15
17
|
DEFAULT_FLUSH_SIZE = 100
|
16
18
|
DEFAULT_COMMIT_WITH_FLUSH = true
|
17
19
|
|
@@ -42,9 +44,13 @@ module Fluent
|
|
42
44
|
config_param :unique_key_field, :string, :default => nil,
|
43
45
|
:desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
|
44
46
|
config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
|
45
|
-
:desc => 'A field name of fluentd tag in the Solr schema.xml (default
|
46
|
-
config_param :
|
47
|
-
:desc => 'A field name of event timestamp in the Solr schema.xml (default
|
47
|
+
:desc => 'A field name of fluentd tag in the Solr schema.xml (default tag).'
|
48
|
+
config_param :time_field, :string, :default => DEFAULT_TIME_FIELD,
|
49
|
+
:desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
|
50
|
+
config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
|
51
|
+
:desc => 'The format of the time field (default %d/%b/%Y:%H:%M:%S %z).'
|
52
|
+
config_param :time_output_format, :string, :default => DEFAULT_TIME_OUTPUT_FORMAT,
|
53
|
+
:desc => 'The format of the time field of Solr. (default %FT%TZ).'
|
48
54
|
|
49
55
|
config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
|
50
56
|
:desc => 'A number of events to queue up before writing to Solr (default 100).'
|
@@ -109,15 +115,15 @@ module Fluent
|
|
109
115
|
record.merge!({@tag_field => tag})
|
110
116
|
end
|
111
117
|
|
112
|
-
if record.has_key?(@
|
118
|
+
if record.has_key?(@time_field) then
|
113
119
|
begin
|
114
|
-
event_timestamp_dt =
|
115
|
-
record.merge!({@
|
120
|
+
event_timestamp_dt = Time.strptime(record[@time_field], @time_format).to_s
|
121
|
+
record.merge!({@time_field => Time.parse(event_timestamp_dt.to_s).utc.strftime(@time_output_format)})
|
116
122
|
rescue
|
117
|
-
record.merge!({@
|
123
|
+
record.merge!({@time_field => Time.at(time).utc.strftime(@time_output_format)})
|
118
124
|
end
|
119
125
|
else
|
120
|
-
record.merge!({@
|
126
|
+
record.merge!({@time_field => Time.at(time).utc.strftime(@time_output_format)})
|
121
127
|
end
|
122
128
|
|
123
129
|
if @ignore_undefined_fields then
|
@@ -152,6 +158,16 @@ module Fluent
|
|
152
158
|
end
|
153
159
|
end
|
154
160
|
|
161
|
+
#
|
162
|
+
# delete reserved fields
|
163
|
+
# https://cwiki.apache.org/confluence/display/solr/Defining+Fields
|
164
|
+
#
|
165
|
+
record.each_key do |key|
|
166
|
+
if key[0] == '_' and key[-1] == '_' then
|
167
|
+
record.delete(key)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
155
171
|
documents << record
|
156
172
|
|
157
173
|
if documents.count >= @flush_size
|