fluent-plugin-output-solr 1.0.0 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 411fb997709bdee2ba7bdfacddd331dfacd82c0d
4
- data.tar.gz: 481e8990f52260ad3d5ccabb03032ece66c7ad01
3
+ metadata.gz: 1919ebd408f73f5cf3ccd44214586fc15ac91bf1
4
+ data.tar.gz: a8f9e2f80d397a5f0578e407bb8a23027bcc25ae
5
5
  SHA512:
6
- metadata.gz: e52a8fc04fd20bb336ddbcad6c7e12bf562180c52565736ba149334ba23afd61c8ceea989776d4ad2abb1d4fe8b37bcd0ac1bb4c989e5f251297e44b0328a776
7
- data.tar.gz: 833671aa903c55254b03bf50ed4156f1e9b8a23e17027972a234991f14f2cc1176a6262777315aa4f7612ba7e296e9cff1b589cd01b625dac70a8a474bb24f69
6
+ metadata.gz: ab70f58bc5431a57e151e50ccfad286da2f4df3b911359c83f427a1b5806b39ec9759789c069151a560714eb643ee21fe17c34081bba838f06c594e32882338d
7
+ data.tar.gz: 49327b34e288c8f42f4d0aa93558b56c99b93b59419aa9fa1c31ee540544ed37d2e24423f3d90020941ef9d0ee56c358f3959073d8fc6bf6523581781ef9c6fb
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Fluent::Plugin::SolrOutput
2
2
 
3
- This is a [Fluentd](http://fluentd.org/) output plugin for send data to [Apache Solr](http://lucene.apache.org/solr/). It support [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud) not only Standalone Solr.
3
+ This is a [Fluentd](http://fluentd.org/) output plugin for sending data to [Apache Solr](http://lucene.apache.org/solr/). It supports standalone Solr and [SolrCloud](https://cwiki.apache.org/confluence/display/solr/SolrCloud).
4
4
 
5
5
  ## Requirements
6
6
 
@@ -72,6 +72,14 @@ Ignore undefined fields in the Solr schema.xml.
72
72
  ignore_undefined_fields false
73
73
  ```
74
74
 
75
+ ### string_field_value_max_length
76
+
77
+ A string field value max length. If set -1, it means unlimited (default -1). However, there is a limit of Solr.
78
+
79
+ ```
80
+ string_field_value_max_length -1
81
+ ```
82
+
75
83
  ### unique_key_field
76
84
 
77
85
  A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.
@@ -82,10 +90,10 @@ unique_key_field id
82
90
 
83
91
  ### timestamp_field
84
92
 
85
- A field name of event timestamp in the Solr schema.xml (default event_timestamp).
93
+ A field name of event timestamp in the Solr schema.xml (default time).
86
94
 
87
95
  ```
88
- timestamp_field event_timestamp
96
+ timestamp_field time
89
97
  ```
90
98
 
91
99
  ### flush_size
@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-output-solr"
7
- spec.version = "1.0.0"
7
+ spec.version = "1.0.1"
8
8
  spec.authors = ["Minoru Osuka"]
9
9
  spec.email = ["minoru.osuka@gmail.com"]
10
10
 
11
11
  spec.summary = "Fluent output plugin for sending data to Apache Solr."
12
- spec.description = "Fluent output plugin for sending data to Apache Solr. It support SolrCloud not only Standalone Solr."
12
+ spec.description = "Fluent output plugin for sending data to Apache Solr. It supports standalone Solr and SolrCloud."
13
13
  spec.homepage = "https://github.com/mosuka/fluent-plugin-output-solr"
14
14
 
15
15
  spec.license = "Apache-2.0"
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
24
24
  spec.add_runtime_dependency 'rsolr', '~> 1.0.12'
25
25
  spec.add_runtime_dependency 'zk', '~> 1.9.5'
26
26
 
27
- spec.add_development_dependency 'bundler', '~> 1.14.6'
27
+ spec.add_development_dependency 'bundler', '~> 1.15.1'
28
28
  spec.add_development_dependency 'rake', '~> 11.1.2'
29
29
  spec.add_development_dependency 'test-unit', '~> 3.1.5'
30
30
  spec.add_development_dependency 'minitest', '~> 5.8.3'
@@ -12,8 +12,9 @@ module Fluent::Plugin
12
12
 
13
13
  DEFAULT_COLLECTION = 'collection1'
14
14
  DEFAULT_IGNORE_UNDEFINED_FIELDS = false
15
+ DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
15
16
  DEFAULT_TAG_FIELD = 'tag'
16
- DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
17
+ DEFAULT_TIMESTAMP_FIELD = 'time'
17
18
  DEFAULT_FLUSH_SIZE = 100
18
19
  DEFAULT_BUFFER_TYPE = "memory"
19
20
  DEFAULT_COMMIT_WITH_FLUSH = true
@@ -36,13 +37,15 @@ module Fluent::Plugin
36
37
  :desc => 'The defined fields in the Solr schema.xml. If omitted, it will get fields via Solr Schema API.'
37
38
  config_param :ignore_undefined_fields, :bool, :default => DEFAULT_IGNORE_UNDEFINED_FIELDS,
38
39
  :desc => 'Ignore undefined fields in the Solr schema.xml.'
40
+ config_param :string_field_value_max_length, :integer, :default => DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH,
41
+ :desc => 'Field value max length.'
39
42
 
40
43
  config_param :unique_key_field, :string, :default => nil,
41
44
  :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
42
45
  config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
43
- :desc => 'A field name of fluentd tag in the Solr schema.xml (default event_timestamp).'
46
+ :desc => 'A field name of fluentd tag in the Solr schema.xml (default time).'
44
47
  config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
45
- :desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
48
+ :desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
46
49
 
47
50
  config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
48
51
  :desc => 'A number of events to queue up before writing to Solr (default 100).'
@@ -62,6 +65,7 @@ module Fluent::Plugin
62
65
  def configure(conf)
63
66
  compat_parameters_convert(conf, :inject)
64
67
  super
68
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
65
69
  end
66
70
 
67
71
  def start
@@ -142,6 +146,30 @@ module Fluent::Plugin
142
146
  end
143
147
  end
144
148
 
149
+ if @string_field_value_max_length >= 0 then
150
+ record.each_key do |key|
151
+ if record[key].instance_of?(Array) then
152
+ values = []
153
+ record[key].each do |value|
154
+ if value.instance_of?(String) then
155
+ if value.length > @string_field_value_max_length then
156
+ log.warn "#{key} is too long (#{value.length}, max is #{@string_field_value_max_length})."
157
+ values.push(value.slice(0, @string_field_value_max_length))
158
+ else
159
+ values.push(value)
160
+ end
161
+ end
162
+ end
163
+ record[key] = values
164
+ elsif record[key].instance_of?(String) then
165
+ if record[key].length > @string_field_value_max_length then
166
+ log.warn "#{key} is too long (#{record[key].length}, max is #{@string_field_value_max_length})."
167
+ record[key] = record[key].slice(0, @string_field_value_max_length)
168
+ end
169
+ end
170
+ end
171
+ end
172
+
145
173
  documents << record
146
174
 
147
175
  if documents.count >= @flush_size
@@ -159,7 +187,7 @@ module Fluent::Plugin
159
187
  log.debug "Added %d document(s) to Solr" % documents.count
160
188
  elsif @mode == MODE_SOLRCLOUD then
161
189
  @solr.add documents, collection: @collection, :params => {:commit => @commit_with_flush}
162
- log.debug "Update: Added %d document(s) to Solr" % documents.count
190
+ log.debug "Added #{documents.count} document(s) to Solr"
163
191
  end
164
192
  rescue Exception => e
165
193
  log.warn "Update: An error occurred while indexing: #{e.message}"
@@ -180,7 +208,7 @@ module Fluent::Plugin
180
208
  return unique_key
181
209
 
182
210
  rescue Exception => e
183
- log.warn "Unique key: #{e.message}"
211
+ log.warn "An error occurred: #{e.message}"
184
212
  end
185
213
 
186
214
  def get_fields
@@ -201,7 +229,7 @@ module Fluent::Plugin
201
229
  return fields
202
230
 
203
231
  rescue Exception => e
204
- log.warn "Fields: #{e.message}"
232
+ log.warn "An error occurred: #{e.message}"
205
233
  end
206
234
  end
207
235
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-output-solr
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Minoru Osuka
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-03-29 00:00:00.000000000 Z
11
+ date: 2017-06-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -78,14 +78,14 @@ dependencies:
78
78
  requirements:
79
79
  - - "~>"
80
80
  - !ruby/object:Gem::Version
81
- version: 1.14.6
81
+ version: 1.15.1
82
82
  type: :development
83
83
  prerelease: false
84
84
  version_requirements: !ruby/object:Gem::Requirement
85
85
  requirements:
86
86
  - - "~>"
87
87
  - !ruby/object:Gem::Version
88
- version: 1.14.6
88
+ version: 1.15.1
89
89
  - !ruby/object:Gem::Dependency
90
90
  name: rake
91
91
  requirement: !ruby/object:Gem::Requirement
@@ -156,8 +156,8 @@ dependencies:
156
156
  - - "~>"
157
157
  - !ruby/object:Gem::Version
158
158
  version: 1.1.8
159
- description: Fluent output plugin for sending data to Apache Solr. It support SolrCloud
160
- not only Standalone Solr.
159
+ description: Fluent output plugin for sending data to Apache Solr. It supports standalone
160
+ Solr and SolrCloud.
161
161
  email:
162
162
  - minoru.osuka@gmail.com
163
163
  executables: []