fluent-plugin-groonga 1.0.9 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/doc/text/news.md +9 -1
- data/fluent-plugin-groonga.gemspec +1 -1
- data/lib/fluent/plugin/in_groonga.rb +2 -1
- data/lib/fluent/plugin/out_groonga.rb +18 -0
- data/sample/store-syslog.conf +21 -9
- data/test/output/test_type_guesser.rb +14 -0
- data/test/test_input.rb +1 -1
- data/test/test_output.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1d63f8c4f3214cc7b8c43876faadcb3be1aae8bf
|
4
|
+
data.tar.gz: 0ffcbf7f152c8824991d50616cb277ec88396ac7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6e91d9055541ceb9ca4adf8a695f3f6752400860c9bd765647a841c378e1e885037db339356758993604ec7a60b3a40af66252ff0d8eecde7c5c714546c1984e
|
7
|
+
data.tar.gz: 4d83595cf18cfc73fb60332ba235e5f38814df328cb2c9c624bafc69596c3bdb9d7052db2e364163d1108cf1a2fdddd499d24bac9845018b1ff5f330da377e4a
|
data/doc/text/news.md
CHANGED
@@ -2,9 +2,17 @@
|
|
2
2
|
|
3
3
|
# News
|
4
4
|
|
5
|
+
## 1.1.0: 2016-01-24
|
6
|
+
|
7
|
+
### Improvements
|
8
|
+
|
9
|
+
* out: Stopped to try to create pseudo columns such as `_key`.
|
10
|
+
* out: Supported boolean value.
|
11
|
+
* in: Supported `plugin_register` and `plugin_unregister`.
|
12
|
+
|
5
13
|
## 1.0.9: 2014-11-20
|
6
14
|
|
7
|
-
###
|
15
|
+
### Improvements
|
8
16
|
|
9
17
|
* out: Added log message with host, port and command name on Groonga
|
10
18
|
command execution error.
|
@@ -17,7 +17,7 @@
|
|
17
17
|
|
18
18
|
Gem::Specification.new do |spec|
|
19
19
|
spec.name = "fluent-plugin-groonga"
|
20
|
-
spec.version = "1.0
|
20
|
+
spec.version = "1.1.0"
|
21
21
|
spec.authors = ["Kouhei Sutou"]
|
22
22
|
spec.email = ["kou@clear-code.com"]
|
23
23
|
spec.summary = "Fluentd plugin to store data into Groonga and implement Groonga replication system."
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
#
|
3
|
-
# Copyright (C) 2012-
|
3
|
+
# Copyright (C) 2012-2016 Kouhei Sutou <kou@clear-code.com>
|
4
4
|
#
|
5
5
|
# This library is free software; you can redistribute it and/or
|
6
6
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -87,6 +87,7 @@ module Fluent
|
|
87
87
|
/\Atable_/,
|
88
88
|
/\Acolumn_/,
|
89
89
|
"delete",
|
90
|
+
/\Aplugin_/,
|
90
91
|
"register",
|
91
92
|
"truncate",
|
92
93
|
"load",
|
@@ -296,6 +296,7 @@ module Fluent
|
|
296
296
|
nonexistent_columns = {}
|
297
297
|
records.each do |record|
|
298
298
|
record.each do |key, value|
|
299
|
+
next if pseudo_column_name?(key)
|
299
300
|
column = @columns[key]
|
300
301
|
if column.nil?
|
301
302
|
nonexistent_columns[key] ||= []
|
@@ -345,6 +346,10 @@ module Fluent
|
|
345
346
|
end
|
346
347
|
end
|
347
348
|
|
349
|
+
def pseudo_column_name?(name)
|
350
|
+
name.start_with?("_")
|
351
|
+
end
|
352
|
+
|
348
353
|
def create_column(name, sample_values)
|
349
354
|
mapping = @mappings.find do |mapping|
|
350
355
|
mapping.name == name
|
@@ -403,6 +408,7 @@ module Fluent
|
|
403
408
|
end
|
404
409
|
|
405
410
|
def guess
|
411
|
+
return "Bool" if bool_values?
|
406
412
|
return "Time" if time_values?
|
407
413
|
return "Int32" if int32_values?
|
408
414
|
return "Int64" if int64_values?
|
@@ -437,6 +443,18 @@ module Fluent
|
|
437
443
|
end
|
438
444
|
end
|
439
445
|
|
446
|
+
BOOL_VALUES = [
|
447
|
+
true,
|
448
|
+
false,
|
449
|
+
"true",
|
450
|
+
"false",
|
451
|
+
]
|
452
|
+
def bool_values?
|
453
|
+
@sample_values.all? do |sample_value|
|
454
|
+
BOOL_VALUES.include?(sample_value)
|
455
|
+
end
|
456
|
+
end
|
457
|
+
|
440
458
|
def time_values?
|
441
459
|
now = Time.now.to_i
|
442
460
|
year_in_seconds = 365 * 24 * 60 * 60
|
data/sample/store-syslog.conf
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
# Required plugins:
|
2
|
-
# * fluent-plugin-config-expander
|
3
2
|
# * fluent-plugin-forest
|
4
3
|
# * fluent-plugin-parser
|
5
4
|
# * fluent-plugin-record-reformer
|
@@ -10,14 +9,11 @@
|
|
10
9
|
</source>
|
11
10
|
|
12
11
|
<source>
|
13
|
-
type
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
tag raw.messages.log.${hostname}
|
19
|
-
format none
|
20
|
-
</config>
|
12
|
+
type tail
|
13
|
+
path /var/log/messages
|
14
|
+
pos_file /var/log/td-agent/messages.pos
|
15
|
+
tag "raw.messages.log.#{Socket.gethostname}"
|
16
|
+
format none
|
21
17
|
</source>
|
22
18
|
|
23
19
|
<match raw.*.log.**>
|
@@ -66,12 +62,28 @@
|
|
66
62
|
normalizer NormalizerAuto
|
67
63
|
</table>
|
68
64
|
|
65
|
+
<table>
|
66
|
+
name Hosts
|
67
|
+
flags TABLE_PAT_KEY
|
68
|
+
key_type ShortText
|
69
|
+
# normalizer NormalizerAuto
|
70
|
+
</table>
|
71
|
+
|
69
72
|
<table>
|
70
73
|
name Timestamps
|
71
74
|
flags TABLE_PAT_KEY
|
72
75
|
key_type Time
|
73
76
|
</table>
|
74
77
|
|
78
|
+
<mapping>
|
79
|
+
name host
|
80
|
+
type Hosts
|
81
|
+
<index>
|
82
|
+
table Hosts
|
83
|
+
name logs_index
|
84
|
+
</index>
|
85
|
+
</mapping>
|
86
|
+
|
75
87
|
<mapping>
|
76
88
|
name timestamp
|
77
89
|
type Time
|
@@ -22,6 +22,20 @@ class OutputTypeGuesserTest < Test::Unit::TestCase
|
|
22
22
|
guesser.guess
|
23
23
|
end
|
24
24
|
|
25
|
+
sub_test_case "Bool" do
|
26
|
+
test "true" do
|
27
|
+
assert_equal("Bool", guess([true]))
|
28
|
+
end
|
29
|
+
|
30
|
+
test "false" do
|
31
|
+
assert_equal("Bool", guess([false]))
|
32
|
+
end
|
33
|
+
|
34
|
+
test "string" do
|
35
|
+
assert_equal("Bool", guess(["true", "false"]))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
25
39
|
sub_test_case "Time" do
|
26
40
|
test "now" do
|
27
41
|
now = Time.now.to_i
|
data/test/test_input.rb
CHANGED
data/test/test_output.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-groonga
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -186,15 +186,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
186
186
|
version: '0'
|
187
187
|
requirements: []
|
188
188
|
rubyforge_project:
|
189
|
-
rubygems_version: 2.
|
189
|
+
rubygems_version: 2.4.5.1
|
190
190
|
signing_key:
|
191
191
|
specification_version: 4
|
192
192
|
summary: Fluentd plugin to store data into Groonga and implement Groonga replication
|
193
193
|
system.
|
194
194
|
test_files:
|
195
|
-
- test/test_output.rb
|
196
195
|
- test/test_input.rb
|
197
196
|
- test/run-test.rb
|
197
|
+
- test/test_output.rb
|
198
198
|
- test/output/test_table_index_definition.rb
|
199
199
|
- test/output/test_type_guesser.rb
|
200
200
|
- test/output/test_table_definition.rb
|