fluent-plugin-output-solr 0.4.9 → 0.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef294b47232fadcb70f4fb7c0a76241be7876a3e
-  data.tar.gz: 8f181b80cca4ef931af172eae0af62d5f949b9fe
+  metadata.gz: fb2ceab3cbbcea52fc45b798fcafc0101e0d0477
+  data.tar.gz: e68164616412a7a22e2dfd0a7c480b36cfbb9e33
 SHA512:
-  metadata.gz: 18a6d6d41beece8a5c19b0448aad55fd915de89de9225a5577a747dc5c2e09eaba1decc623f44dc04b36e6a245d26f4393b800ef344a754323ac157f056532c0
-  data.tar.gz: f32d2d636b045c2187d2c03052ee2caa963f3812a106507fdb79e0a32dfebfeef9b9fd2972a18ec9cc27a8f5f62f6e334cda56767b16d8ba31e3b5e8ef05e355
+  metadata.gz: 88eedfd9043ffc95485489c43e96a7bbda53149c56f62ba515b40fe66edc1fda5765cfa98f4466c50bc5316908c8cf133a6ba706d314a8fe5a631a662094d4d4
+  data.tar.gz: be00852de009eacfc458478a5a25c313ddf318750e050925136fac4325ecdde1a90d017cc806087ad98e77aa92b6269d67464d0668948401c71929c60a6ca1dd

data/README.md

@@ -88,12 +88,28 @@ A string field value max length. If set -1, it means unlimited (default -1). How
 string_field_value_max_length -1
 ```
 
-### timestamp_field
+### time_field
 
-A field name of event timestamp in the Solr schema.xml (default event_timestamp).
+A field name of event timestamp in the Solr schema.xml (default time).
 
 ```
-timestamp_field event_timestamp
+time_field time
+```
+
+### time_format
+
+The format of the time field (default %FT%TZ).
+
+```
+time_format %FT%TZ
+```
+
+### time_output_format
+
+The format of the time field of Solr. (default %FT%TZ).
+
+```
+time_output_format %FT%TZ
 ```
 
 ### flush_size

data/fluent-plugin-output-solr.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-output-solr"
-  spec.version       = "0.4.9"
+  spec.version       = "0.4.10"
   spec.authors       = ["Minoru Osuka"]
   spec.email         = ["minoru.osuka@gmail.com"]
 

data/lib/fluent/plugin/out_solr.rb

@@ -11,7 +11,9 @@ module Fluent
     DEFAULT_IGNORE_UNDEFINED_FIELDS = false
     DEFAULT_STRING_FIELD_VALUE_MAX_LENGTH = -1
     DEFAULT_TAG_FIELD = 'tag'
-    DEFAULT_TIMESTAMP_FIELD = 'event_timestamp'
+    DEFAULT_TIME_FIELD = 'time'
+    DEFAULT_TIME_FORMAT = '%FT%TZ'
+    DEFAULT_TIME_OUTPUT_FORMAT = '%FT%TZ'
     DEFAULT_FLUSH_SIZE = 100
     DEFAULT_COMMIT_WITH_FLUSH = true
 
@@ -42,9 +44,13 @@ module Fluent
     config_param :unique_key_field, :string, :default => nil,
                  :desc => 'A field name of unique key in the Solr schema.xml. If omitted, it will get unique key via Solr Schema API.'
     config_param :tag_field, :string, :default => DEFAULT_TAG_FIELD,
-                 :desc => 'A field name of fluentd tag in the Solr schema.xml (default event_timestamp).'
-    config_param :timestamp_field, :string, :default => DEFAULT_TIMESTAMP_FIELD,
-                 :desc => 'A field name of event timestamp in the Solr schema.xml (default event_timestamp).'
+                 :desc => 'A field name of fluentd tag in the Solr schema.xml (default tag).'
+    config_param :time_field, :string, :default => DEFAULT_TIME_FIELD,
+                 :desc => 'A field name of event timestamp in the Solr schema.xml (default time).'
+    config_param :time_format, :string, :default => DEFAULT_TIME_FORMAT,
+                 :desc => 'The format of the time field (default %d/%b/%Y:%H:%M:%S %z).'
+    config_param :time_output_format, :string, :default => DEFAULT_TIME_OUTPUT_FORMAT,
+                 :desc => 'The format of the time field of Solr. (default %FT%TZ).'
 
     config_param :flush_size, :integer, :default => DEFAULT_FLUSH_SIZE,
                  :desc => 'A number of events to queue up before writing to Solr (default 100).'
@@ -109,15 +115,15 @@ module Fluent
           record.merge!({@tag_field => tag})
         end
 
-        if record.has_key?(@timestamp_field) then
+        if record.has_key?(@time_field) then
           begin
-            event_timestamp_dt = DateTime.strptime(record[@timestamp_field], "%d/%b/%Y:%H:%M:%S %z").to_s
-            record.merge!({@timestamp_field => Time.parse(event_timestamp_dt.to_s).utc.strftime('%FT%TZ')})
+            event_timestamp_dt = Time.strptime(record[@time_field], @time_format).to_s
+            record.merge!({@time_field => Time.parse(event_timestamp_dt.to_s).utc.strftime(@time_output_format)})
           rescue
-            record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
+            record.merge!({@time_field => Time.at(time).utc.strftime(@time_output_format)})
           end
         else
-          record.merge!({@timestamp_field => Time.at(time).utc.strftime('%FT%TZ')})
+          record.merge!({@time_field => Time.at(time).utc.strftime(@time_output_format)})
         end
 
         if @ignore_undefined_fields then
@@ -152,6 +158,16 @@ module Fluent
           end
         end
 
+        #
+        # delete reserved fields
+        # https://cwiki.apache.org/confluence/display/solr/Defining+Fields
+        #
+        record.each_key do |key|
+          if key[0] == '_' and key[-1] == '_' then
+            record.delete(key)
+          end
+        end
+
         documents << record
 
         if documents.count >= @flush_size

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-output-solr
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.4.10
 platform: ruby
 authors:
 - Minoru Osuka