fluent-plugin-groonga 1.0.3 → 1.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +120 -8
  3. data/doc/text/news.md +15 -0
  4. data/fluent-plugin-groonga.gemspec +5 -5
  5. data/lib/fluent/plugin/in_groonga.rb +32 -12
  6. data/lib/fluent/plugin/out_groonga.rb +300 -129
  7. data/sample/gqtp.conf +2 -2
  8. data/sample/store.conf +15 -0
  9. data/test/output/test_type_guesser.rb +127 -0
  10. data/test/test_output.rb +70 -30
  11. metadata +43 -38
data/sample/gqtp.conf CHANGED
@@ -3,13 +3,13 @@
3
3
  protocol gqtp
4
4
 
5
5
  real_host 127.0.0.1
6
- real_port 20041
6
+ real_port 20043
7
7
  </source>
8
8
 
9
9
  <match groonga.command.*>
10
10
  type groonga
11
11
  protocol gqtp
12
12
  host 127.0.0.1
13
- port 30041
13
+ port 30043
14
14
  flush_interval 1
15
15
  </match>
data/sample/store.conf ADDED
@@ -0,0 +1,15 @@
1
+ <source>
2
+ type forward
3
+ </source>
4
+
5
+ <match log.*>
6
+ type groonga
7
+ table Logs
8
+
9
+ protocol http
10
+ host 127.0.0.1
11
+
12
+ buffer_type file
13
+ buffer_path /tmp/buffer
14
+ flush_interval 1
15
+ </match>
data/test/output/test_type_guesser.rb ADDED
@@ -0,0 +1,127 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "fluent/plugin/out_groonga"
17
+
18
+ class OutputTypeGuesserTest < Test::Unit::TestCase
19
+ sub_test_case "#guess" do
20
+ def guess(sample_values)
21
+ guesser = Fluent::GroongaOutput::Schema::TypeGuesser.new(sample_values)
22
+ guesser.guess
23
+ end
24
+
25
+ sub_test_case "Time" do
26
+ test "now" do
27
+ now = Time.now.to_i
28
+ assert_equal("Time", guess([now]))
29
+ end
30
+
31
+ test "past value" do
32
+ now = Time.now.to_i
33
+ year_in_seconds = 365 * 24 * 60 * 60
34
+ past = now - (9 * year_in_seconds)
35
+ assert_equal("Time", guess([past]))
36
+ end
37
+
38
+ test "future value" do
39
+ now = Time.now.to_i
40
+ year_in_seconds = 365 * 24 * 60 * 60
41
+ future = now + (9 * year_in_seconds)
42
+ assert_equal("Time", guess([future]))
43
+ end
44
+
45
+ test "all type values" do
46
+ now = Time.now.to_i
47
+ year_in_seconds = 365 * 24 * 60 * 60
48
+ past = now - (9 * year_in_seconds)
49
+ future = now + (9 * year_in_seconds)
50
+ assert_equal("Time", guess([now, past, future]))
51
+ end
52
+ end
53
+
54
+ sub_test_case "Int32" do
55
+ test "min" do
56
+ int32_min = -(2 ** 31)
57
+ assert_equal("Int32", guess([int32_min]))
58
+ end
59
+
60
+ test "max" do
61
+ int32_max = 2 ** 31 - 1
62
+ assert_equal("Int32", guess([int32_max]))
63
+ end
64
+
65
+ test "zero" do
66
+ assert_equal("Int32", guess([0]))
67
+ end
68
+
69
+ test "string" do
70
+ assert_equal("Int32", guess(["0"]))
71
+ end
72
+ end
73
+
74
+ sub_test_case "Int64" do
75
+ test "int32_min - 1" do
76
+ int32_min = -(2 ** 31)
77
+ assert_equal("Int64", guess([int32_min - 1]))
78
+ end
79
+
80
+ test "int32_max + 1" do
81
+ int32_max = 2 ** 31 - 1
82
+ assert_equal("Int64", guess([int32_max + 1]))
83
+ end
84
+
85
+ test "string" do
86
+ assert_equal("Int64", guess([(2 ** 32).to_s]))
87
+ end
88
+ end
89
+
90
+ sub_test_case "Float" do
91
+ test "positive" do
92
+ assert_equal("Float", guess([1.0]))
93
+ end
94
+
95
+ test "negative" do
96
+ assert_equal("Float", guess([-1.0]))
97
+ end
98
+
99
+ test "zero" do
100
+ assert_equal("Float", guess([0.0]))
101
+ end
102
+
103
+ test "string" do
104
+ assert_equal("Float", guess(["1.1"]))
105
+ end
106
+ end
107
+
108
+ sub_test_case "WGS84GeoPoint" do
109
+ test "\#{LATITUDE},\#{LONGITUDE}" do
110
+ statue_of_liberty = "40.689167,-74.044444"
111
+ assert_equal("WGS84GeoPoint", guess([statue_of_liberty]))
112
+ end
113
+
114
+ test "\#{LATITUDE}x\#{LONGITUDE}" do
115
+ statue_of_liberty = "40.689167x-74.044444"
116
+ assert_equal("WGS84GeoPoint", guess([statue_of_liberty]))
117
+ end
118
+ end
119
+
120
+ sub_test_case "Text" do
121
+ test "message" do
122
+ message = "failed to load data"
123
+ assert_equal("Text", guess([message]))
124
+ end
125
+ end
126
+ end
127
+ end
data/test/test_output.rb CHANGED
@@ -47,43 +47,43 @@ EOC
47
47
  class HTTPTest < self
48
48
  setup :before => :append
49
49
  def setup_real_server
50
+ @request_parser = HTTP::Parser.new
51
+ @request_body = ""
52
+ @response_body = nil
53
+
50
54
  @real_host = "127.0.0.1"
51
55
  @real_port = 29292
52
- @real_server_pid = fork do
53
- exit
54
- real_server = TCPServer.new(@real_host, @real_port)
55
- response_config = WEBrick::Config::HTTP.dup.update(:Logger => $log)
56
- real_response = WEBrick::HTTPResponse.new(response_config)
57
- request_headers = nil
58
- request_body = ""
59
- client = real_server.accept
60
- real_server.close
61
- parser = HTTP::Parser.new
62
- parser.on_headers_complete = lambda do |headers|
63
- request_headers = headers
64
- end
65
- parser.on_body = lambda do |chunk|
66
- request_body << chunk
67
- end
68
- parser.on_message_complete = lambda do
69
- real_response.send_response(client)
70
- client.close
71
- end
72
-
56
+ @real_server_thread = Thread.new do
57
+ @real_server = TCPServer.new(@real_host, @real_port)
73
58
  loop do
74
- break if client.closed?
75
- data = client.readpartial(4096)
76
- break if data.nil?
77
- parser << data
59
+ response_config = WEBrick::Config::HTTP.dup.update(:Logger => $log)
60
+ real_response = WEBrick::HTTPResponse.new(response_config)
61
+ request_headers = nil
62
+ request_body = ""
63
+ client = @real_server.accept
64
+ @request_parser.on_body = lambda do |chunk|
65
+ @request_body << chunk
66
+ end
67
+ @request_parser.on_message_complete = lambda do
68
+ real_response.body = @response_body
69
+ real_response.send_response(client)
70
+ client.close
71
+ end
72
+
73
+ loop do
74
+ break if client.closed?
75
+ data = client.readpartial(4096)
76
+ break if data.nil?
77
+ @request_parser << data
78
+ end
78
79
  end
79
80
  end
80
81
  end
81
82
 
82
83
  teardown
83
84
  def teardown_real_server
84
- Process.kill(:INT, @real_server_pid)
85
- Process.kill(:KILL, @real_server_pid)
86
- Process.waitpid(@real_server_pid)
85
+ @real_server_thread.kill
86
+ @real_server.close
87
87
  end
88
88
 
89
89
  def configuration
@@ -96,13 +96,53 @@ EOC
96
96
 
97
97
  class CommandTest < self
98
98
  def test_basic_command
99
+ @response_body = JSON.generate([[0, 0.0, 0.0], true])
99
100
  driver = create_driver("groonga.command.table_create")
100
101
  time = Time.parse("2012-10-26T08:45:42Z").to_i
101
102
  driver.run do
102
103
  driver.emit({"name" => "Users"}, time)
103
104
  end
104
- # p @request_headers
105
- # p @request_body
105
+ assert_equal("/d/table_create?name=Users",
106
+ @request_parser.request_url)
107
+ end
108
+ end
109
+
110
+ class StoreTest < self
111
+ def configuration
112
+ <<-CONFIGURATION
113
+ #{super}
114
+ table Logs
115
+ CONFIGURATION
116
+ end
117
+
118
+ def test_one_message
119
+ @response_body = JSON.generate([[0, 0.0, 0.0], [1]])
120
+ driver = create_driver("log")
121
+ time = Time.parse("2012-10-26T08:45:42Z").to_i
122
+ driver.run do
123
+ driver.emit({"message" => "1st message"}, time)
124
+ end
125
+ assert_equal("/d/load?table=Logs",
126
+ @request_parser.request_url)
127
+ assert_equal([{"message" => "1st message"}],
128
+ JSON.parse(@request_body))
129
+ end
130
+
131
+ def test_multiple_messages
132
+ @response_body = JSON.generate([[0, 0.0, 0.0], [2]])
133
+ driver = create_driver("log")
134
+ time = Time.parse("2012-10-26T08:45:42Z").to_i
135
+ driver.run do
136
+ driver.emit({"message" => "1st message"}, time)
137
+ driver.emit({"message" => "2nd message"}, time + 1)
138
+ end
139
+ assert_equal("/d/load?table=Logs",
140
+ @request_parser.request_url)
141
+ assert_equal([
142
+ {"message" => "1st message"},
143
+ {"message" => "2nd message"},
144
+ ],
145
+ JSON.parse(@request_body))
106
146
  end
107
147
  end
108
148
  end
metadata CHANGED
@@ -1,164 +1,167 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-groonga
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-09-29 00:00:00.000000000 Z
11
+ date: 2014-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: gqtp
28
+ name: groonga-client
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: 1.0.3
33
+ version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 1.0.3
40
+ version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: groonga-command-parser
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: bundler
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: packnga
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - '>='
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: 0.9.6
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - '>='
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: 0.9.6
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: test-unit
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - '>='
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '0'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - '>='
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: test-unit-notify
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - '>='
115
+ - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '0'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - '>='
122
+ - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: redcarpet
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
- - - '>='
129
+ - - ">="
130
130
  - !ruby/object:Gem::Version
131
131
  version: '0'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - '>='
136
+ - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '0'
139
- description: Groonga users can replicate their data by fluent-plugin-groonga
139
+ description: There are two usages. 1) Store data into Groonga. 2) Implement Groonga
140
+ replication system. See documentation for details.
140
141
  email:
141
142
  - kou@clear-code.com
142
143
  executables: []
143
144
  extensions: []
144
145
  extra_rdoc_files: []
145
146
  files:
146
- - README.md
147
+ - ".yardopts"
147
148
  - Gemfile
149
+ - README.md
150
+ - doc/text/configuration.md
151
+ - doc/text/constitution.md
152
+ - doc/text/lgpl-2.1.txt
153
+ - doc/text/news.md
148
154
  - fluent-plugin-groonga.gemspec
149
- - .yardopts
150
- - lib/fluent/plugin/out_groonga.rb
151
155
  - lib/fluent/plugin/in_groonga.rb
156
+ - lib/fluent/plugin/out_groonga.rb
157
+ - sample/command.conf
152
158
  - sample/gqtp.conf
153
159
  - sample/http.conf
154
- - sample/command.conf
155
- - doc/text/configuration.md
156
- - doc/text/news.md
157
- - doc/text/constitution.md
158
- - doc/text/lgpl-2.1.txt
159
- - test/test_output.rb
160
+ - sample/store.conf
161
+ - test/output/test_type_guesser.rb
160
162
  - test/run-test.rb
161
163
  - test/test_input.rb
164
+ - test/test_output.rb
162
165
  homepage: https://github.com/groonga/fluent-plugin-groonga
163
166
  licenses:
164
167
  - LGPL-2.1
@@ -169,22 +172,24 @@ require_paths:
169
172
  - lib
170
173
  required_ruby_version: !ruby/object:Gem::Requirement
171
174
  requirements:
172
- - - '>='
175
+ - - ">="
173
176
  - !ruby/object:Gem::Version
174
177
  version: '0'
175
178
  required_rubygems_version: !ruby/object:Gem::Requirement
176
179
  requirements:
177
- - - '>='
180
+ - - ">="
178
181
  - !ruby/object:Gem::Version
179
182
  version: '0'
180
183
  requirements: []
181
184
  rubyforge_project:
182
- rubygems_version: 2.0.7
185
+ rubygems_version: 2.2.2
183
186
  signing_key:
184
187
  specification_version: 4
185
- summary: Fluentd plugin collection for groonga users
188
+ summary: Fluentd plugin to store data into Groonga and implement Groonga replication
189
+ system.
186
190
  test_files:
187
191
  - test/test_output.rb
188
- - test/run-test.rb
189
192
  - test/test_input.rb
193
+ - test/run-test.rb
194
+ - test/output/test_type_guesser.rb
190
195
  has_rdoc: