fluent-plugin-output-solr 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -3
- data/fluent-plugin-output-solr.gemspec +3 -3
- data/lib/fluent/plugin/out_solr.rb +34 -6
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1919ebd408f73f5cf3ccd44214586fc15ac91bf1
|
4
|
+
data.tar.gz: a8f9e2f80d397a5f0578e407bb8a23027bcc25ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab70f58bc5431a57e151e50ccfad286da2f4df3b911359c83f427a1b5806b39ec9759789c069151a560714eb643ee21fe17c34081bba838f06c594e32882338d
|
7
|
+
data.tar.gz: 49327b34e288c8f42f4d0aa93558b56c99b93b59419aa9fa1c31ee540544ed37d2e24423f3d90020941ef9d0ee56c358f3959073d8fc6bf6523581781ef9c6fb
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Fluent::Plugin::SolrOutput
|
2
2
|
|
3
|
-
This is a [Fluentd](http://fluentd.org/) output plugin for
|
3
|
+
This is a [Fluentd](http://fluentd.org/) output plugin for sending data to [Apache Solr](http://lucene.apache.org/solr/). It supports standalone Solr and [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud).
|
4
4
|
|
5
5
|
## Requirements
|
6
6
|
|
@@ -72,6 +72,14 @@ Ignore undefined fields in the Solr schema.xml.
|
|
72
72
|
ignore_undefined_fields false
|
73
73
|
```
|
74
74
|
|
75
|
+
### string_field_value_max_length
|
76
|
+
|
77
|
+
A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
|
78
|
+
|
79
|
+
```
|
80
|
+
string_field_value_max_length -1
|
81
|
+
```
|
82
|
+
|
75
83
|
### unique_key_field
|
76
84
|
|
77
85
|
A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
|
@@ -82,10 +90,10 @@ unique_key_field id
|
|
82
90
|
|
83
91
|
### timestamp_field
|
84
92
|
|
85
|
-
A field name of event timestamp in the Solr schema.xml (default
|
93
|
+
A field name of event timestamp in the Solr schema.xml (default time).
|
86
94
|
|
87
95
|
```
|
88
|
-
timestamp_field
|
96
|
+
timestamp_field time
|
89
97
|
```
|
90
98
|
|
91
99
|
### flush_size
|
@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-output-solr"
|
7
|
-
spec.version = "1.0.
|
7
|
+
spec.version = "1.0.1"
|
8
8
|
spec.authors = ["Minoru Osuka"]
|
9
9
|
spec.email = ["minoru.osuka@gmail.com"]
|
10
10
|
|
11
11
|
spec.summary = "Fluent output plugin for sending data to Apache Solr."
|
12
|
-
spec.description = "Fluent output plugin for sending data to Apache Solr. It
|
12
|
+
spec.description = "Fluent output plugin for sending data to Apache Solr. It supports standalone Solr and SolrCloud."
|
13
13
|
spec.homepage = "https://github.com/mosuka/fluent-plugin-output-solr"
|
14
14
|
|
15
15
|
spec.license = "Apache-2.0"
|
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_runtime_dependency 'rsolr', '~> 1.0.12'
|
25
25
|
spec.add_runtime_dependency 'zk', '~> 1.9.5'
|
26
26
|
|
27
|
-
spec.add_development_dependency 'bundler', '~> 1.
|
27
|
+
spec.add_development_dependency 'bundler', '~> 1.15.1'
|
28
28
|
spec.add_development_dependency 'rake', '~> 11.1.2'
|
29
29
|
spec.add_development_dependency 'test-unit', '~> 3.1.5'
|
30
30
|
spec.add_development_dependency 'minitest', '~> 5.8.3'
|
@@ -12,8 +12,9 @@ module Fluent::Plugin
|
|
12
12
|
|
13
13
|
DEFAULT_COLLECTION = 'collection1'
|
14
14
|
DEFAULT_IGNORE_UNDEFINED_FIELDS = false
|
15
|
+
DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
|
15
16
|
DEFAULT_TAG_FIELD = 'tag'
|
16
|
-
DEFAULT_TIMESTAMP_FIELD = '
|
17
|
+
DEFAULT_TIMESTAMP_FIELD = 'time'
|
17
18
|
DEFAULT_FLUSH_SIZE = 100
|
18
19
|
DEFAULT_BUFFER_TYPE = "memory"
|
19
20
|
DEFAULT_COMMIT_WITH_FLUSH = true
|
@@ -36,13 +37,15 @@ module Fluent::Plugin
|
|
36
37
|
:desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
|
37
38
|
config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
|
38
39
|
:desc => 'Ignore undefined fields in the Solr schema.xml.'
|
40
|
+
config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
|
41
|
+
:desc => 'Field value max length.'
|
39
42
|
|
40
43
|
config_param :unique_key_field, :string, :default => nil,
|
41
44
|
:desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
|
42
45
|
config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
|
43
|
-
:desc => 'A field name of fluentd tag in the Solr schema.xml (default
|
46
|
+
:desc => 'A field name of fluentd tag in the Solr schema.xml (default time).'
|
44
47
|
config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
|
45
|
-
:desc => 'A field name of event timestamp in the Solr schema.xml (default
|
48
|
+
:desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
|
46
49
|
|
47
50
|
config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
|
48
51
|
:desc => 'A number of events to queue up before writing to Solr (default 100).'
|
@@ -62,6 +65,7 @@ module Fluent::Plugin
|
|
62
65
|
def configure(conf)
|
63
66
|
compat_parameters_convert(conf, :inject)
|
64
67
|
super
|
68
|
+
raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
|
65
69
|
end
|
66
70
|
|
67
71
|
def start
|
@@ -142,6 +146,30 @@ module Fluent::Plugin
|
|
142
146
|
end
|
143
147
|
end
|
144
148
|
|
149
|
+
if @string_field_value_max_length >= 0 then
|
150
|
+
record.each_key do |key|
|
151
|
+
if record[key].instance_of?(Array) then
|
152
|
+
values = []
|
153
|
+
record[key].each do |value|
|
154
|
+
if value.instance_of?(String) then
|
155
|
+
if value.length > @string_field_value_max_length then
|
156
|
+
log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
|
157
|
+
values.push(value.slice(0, @string_field_value_max_length))
|
158
|
+
else
|
159
|
+
values.push(value)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
record[key] = values
|
164
|
+
elsif record[key].instance_of?(String) then
|
165
|
+
if record[key].length > @string_field_value_max_length then
|
166
|
+
log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
|
167
|
+
record[key] = record[key].slice(0, @string_field_value_max_length)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
145
173
|
documents << record
|
146
174
|
|
147
175
|
if documents.count >= @flush_size
|
@@ -159,7 +187,7 @@ module Fluent::Plugin
|
|
159
187
|
log.debug "Added %d document(s) to Solr" % documents.count
|
160
188
|
elsif @mode == MODE_SOLRCLOUD then
|
161
189
|
@solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
|
162
|
-
log.debug "
|
190
|
+
log.debug "Added #{documents.count} document(s) to Solr"
|
163
191
|
end
|
164
192
|
rescue Exception => e
|
165
193
|
log.warn "Update: An error occurred while indexing: #{e.message}"
|
@@ -180,7 +208,7 @@ module Fluent::Plugin
|
|
180
208
|
return unique_key
|
181
209
|
|
182
210
|
rescue Exception => e
|
183
|
-
log.warn "
|
211
|
+
log.warn "An error occurred: #{e.message}"
|
184
212
|
end
|
185
213
|
|
186
214
|
def get_fields
|
@@ -201,7 +229,7 @@ module Fluent::Plugin
|
|
201
229
|
return fields
|
202
230
|
|
203
231
|
rescue Exception => e
|
204
|
-
log.warn "
|
232
|
+
log.warn "An error occurred: #{e.message}"
|
205
233
|
end
|
206
234
|
end
|
207
235
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-output-solr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Minoru Osuka
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -78,14 +78,14 @@ dependencies:
|
|
78
78
|
requirements:
|
79
79
|
- - "~>"
|
80
80
|
- !ruby/object:Gem::Version
|
81
|
-
version: 1.
|
81
|
+
version: 1.15.1
|
82
82
|
type: :development
|
83
83
|
prerelease: false
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
85
85
|
requirements:
|
86
86
|
- - "~>"
|
87
87
|
- !ruby/object:Gem::Version
|
88
|
-
version: 1.
|
88
|
+
version: 1.15.1
|
89
89
|
- !ruby/object:Gem::Dependency
|
90
90
|
name: rake
|
91
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,8 +156,8 @@ dependencies:
|
|
156
156
|
- - "~>"
|
157
157
|
- !ruby/object:Gem::Version
|
158
158
|
version: 1.1.8
|
159
|
-
description: Fluent output plugin for sending data to Apache Solr. It
|
160
|
-
|
159
|
+
description: Fluent output plugin for sending data to Apache Solr. It supports standalone
|
160
|
+
Solr and SolrCloud.
|
161
161
|
email:
|
162
162
|
- minoru.osuka@gmail.com
|
163
163
|
executables: []
|