fluent-plugin-groonga 1.0.3 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +120 -8
  3. data/doc/text/news.md +15 -0
  4. data/fluent-plugin-groonga.gemspec +5 -5
  5. data/lib/fluent/plugin/in_groonga.rb +32 -12
  6. data/lib/fluent/plugin/out_groonga.rb +300 -129
  7. data/sample/gqtp.conf +2 -2
  8. data/sample/store.conf +15 -0
  9. data/test/output/test_type_guesser.rb +127 -0
  10. data/test/test_output.rb +70 -30
  11. metadata +43 -38
data/sample/gqtp.conf CHANGED
@@ -3,13 +3,13 @@
3
3
  protocol gqtp
4
4
 
5
5
  real_host 127.0.0.1
6
- real_port 20041
6
+ real_port 20043
7
7
  </source>
8
8
 
9
9
  <match groonga.command.*>
10
10
  type groonga
11
11
  protocol gqtp
12
12
  host 127.0.0.1
13
- port 30041
13
+ port 30043
14
14
  flush_interval 1
15
15
  </match>
data/sample/store.conf ADDED
@@ -0,0 +1,15 @@
1
+ <source>
2
+ type forward
3
+ </source>
4
+
5
+ <match log.*>
6
+ type groonga
7
+ table Logs
8
+
9
+ protocol http
10
+ host 127.0.0.1
11
+
12
+ buffer_type file
13
+ buffer_path /tmp/buffer
14
+ flush_interval 1
15
+ </match>
data/test/output/test_type_guesser.rb ADDED
@@ -0,0 +1,127 @@
1
+ # Copyright (C) 2014 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License version 2.1 as published by the Free Software Foundation.
6
+ #
7
+ # This library is distributed in the hope that it will be useful,
8
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
9
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10
+ # Lesser General Public License for more details.
11
+ #
12
+ # You should have received a copy of the GNU Lesser General Public
13
+ # License along with this library; if not, write to the Free Software
14
+ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
+
16
+ require "fluent/plugin/out_groonga"
17
+
18
+ class OutputTypeGuesserTest < Test::Unit::TestCase
19
+ sub_test_case "#guess" do
20
+ def guess(sample_values)
21
+ guesser = Fluent::GroongaOutput::Schema::TypeGuesser.new(sample_values)
22
+ guesser.guess
23
+ end
24
+
25
+ sub_test_case "Time" do
26
+ test "now" do
27
+ now = Time.now.to_i
28
+ assert_equal("Time", guess([now]))
29
+ end
30
+
31
+ test "past value" do
32
+ now = Time.now.to_i
33
+ year_in_seconds = 365 * 24 * 60 * 60
34
+ past = now - (9 * year_in_seconds)
35
+ assert_equal("Time", guess([past]))
36
+ end
37
+
38
+ test "future value" do
39
+ now = Time.now.to_i
40
+ year_in_seconds = 365 * 24 * 60 * 60
41
+ future = now + (9 * year_in_seconds)
42
+ assert_equal("Time", guess([future]))
43
+ end
44
+
45
+ test "all type values" do
46
+ now = Time.now.to_i
47
+ year_in_seconds = 365 * 24 * 60 * 60
48
+ past = now - (9 * year_in_seconds)
49
+ future = now + (9 * year_in_seconds)
50
+ assert_equal("Time", guess([now, past, future]))
51
+ end
52
+ end
53
+
54
+ sub_test_case "Int32" do
55
+ test "min" do
56
+ int32_min = -(2 ** 31)
57
+ assert_equal("Int32", guess([int32_min]))
58
+ end
59
+
60
+ test "max" do
61
+ int32_max = 2 ** 31 - 1
62
+ assert_equal("Int32", guess([int32_max]))
63
+ end
64
+
65
+ test "zero" do
66
+ assert_equal("Int32", guess([0]))
67
+ end
68
+
69
+ test "string" do
70
+ assert_equal("Int32", guess(["0"]))
71
+ end
72
+ end
73
+
74
+ sub_test_case "Int64" do
75
+ test "int32_min - 1" do
76
+ int32_min = -(2 ** 31)
77
+ assert_equal("Int64", guess([int32_min - 1]))
78
+ end
79
+
80
+ test "int32_max + 1" do
81
+ int32_max = 2 ** 31 - 1
82
+ assert_equal("Int64", guess([int32_max + 1]))
83
+ end
84
+
85
+ test "string" do
86
+ assert_equal("Int64", guess([(2 ** 32).to_s]))
87
+ end
88
+ end
89
+
90
+ sub_test_case "Float" do
91
+ test "positive" do
92
+ assert_equal("Float", guess([1.0]))
93
+ end
94
+
95
+ test "negative" do
96
+ assert_equal("Float", guess([-1.0]))
97
+ end
98
+
99
+ test "zero" do
100
+ assert_equal("Float", guess([0.0]))
101
+ end
102
+
103
+ test "string" do
104
+ assert_equal("Float", guess(["1.1"]))
105
+ end
106
+ end
107
+
108
+ sub_test_case "WGS84GeoPoint" do
109
+ test "\#{LATITUDE},\#{LONGITUDE}" do
110
+ statue_of_liberty = "40.689167,-74.044444"
111
+ assert_equal("WGS84GeoPoint", guess([statue_of_liberty]))
112
+ end
113
+
114
+ test "\#{LATITUDE}x\#{LONGITUDE}" do
115
+ statue_of_liberty = "40.689167x-74.044444"
116
+ assert_equal("WGS84GeoPoint", guess([statue_of_liberty]))
117
+ end
118
+ end
119
+
120
+ sub_test_case "Text" do
121
+ test "message" do
122
+ message = "failed to load data"
123
+ assert_equal("Text", guess([message]))
124
+ end
125
+ end
126
+ end
127
+ end
data/test/test_output.rb CHANGED
@@ -47,43 +47,43 @@ EOC
47
47
  class HTTPTest < self
48
48
  setup :before => :append
49
49
  def setup_real_server
50
+ @request_parser = HTTP::Parser.new
51
+ @request_body = ""
52
+ @response_body = nil
53
+
50
54
  @real_host = "127.0.0.1"
51
55
  @real_port = 29292
52
- @real_server_pid = fork do
53
- exit
54
- real_server = TCPServer.new(@real_host, @real_port)
55
- response_config = WEBrick::Config::HTTP.dup.update(:Logger => $log)
56
- real_response = WEBrick::HTTPResponse.new(response_config)
57
- request_headers = nil
58
- request_body = ""
59
- client = real_server.accept
60
- real_server.close
61
- parser = HTTP::Parser.new
62
- parser.on_headers_complete = lambda do |headers|
63
- request_headers = headers
64
- end
65
- parser.on_body = lambda do |chunk|
66
- request_body << chunk
67
- end
68
- parser.on_message_complete = lambda do
69
- real_response.send_response(client)
70
- client.close
71
- end
72
-
56
+ @real_server_thread = Thread.new do
57
+ @real_server = TCPServer.new(@real_host, @real_port)
73
58
  loop do
74
- break if client.closed?
75
- data = client.readpartial(4096)
76
- break if data.nil?
77
- parser << data
59
+ response_config = WEBrick::Config::HTTP.dup.update(:Logger => $log)
60
+ real_response = WEBrick::HTTPResponse.new(response_config)
61
+ request_headers = nil
62
+ request_body = ""
63
+ client = @real_server.accept
64
+ @request_parser.on_body = lambda do |chunk|
65
+ @request_body << chunk
66
+ end
67
+ @request_parser.on_message_complete = lambda do
68
+ real_response.body = @response_body
69
+ real_response.send_response(client)
70
+ client.close
71
+ end
72
+
73
+ loop do
74
+ break if client.closed?
75
+ data = client.readpartial(4096)
76
+ break if data.nil?
77
+ @request_parser << data
78
+ end
78
79
  end
79
80
  end
80
81
  end
81
82
 
82
83
  teardown
83
84
  def teardown_real_server
84
- Process.kill(:INT, @real_server_pid)
85
- Process.kill(:KILL, @real_server_pid)
86
- Process.waitpid(@real_server_pid)
85
+ @real_server_thread.kill
86
+ @real_server.close
87
87
  end
88
88
 
89
89
  def configuration
@@ -96,13 +96,53 @@ EOC
96
96
 
97
97
  class CommandTest < self
98
98
  def test_basic_command
99
+ @response_body = JSON.generate([[0, 0.0, 0.0], true])
99
100
  driver = create_driver("groonga.command.table_create")
100
101
  time = Time.parse("2012-10-26T08:45:42Z").to_i
101
102
  driver.run do
102
103
  driver.emit({"name" => "Users"}, time)
103
104
  end
104
- # p @request_headers
105
- # p @request_body
105
+ assert_equal("/d/table_create?name=Users",
106
+ @request_parser.request_url)
107
+ end
108
+ end
109
+
110
+ class StoreTest < self
111
+ def configuration
112
+ <<-CONFIGURATION
113
+ #{super}
114
+ table Logs
115
+ CONFIGURATION
116
+ end
117
+
118
+ def test_one_message
119
+ @response_body = JSON.generate([[0, 0.0, 0.0], [1]])
120
+ driver = create_driver("log")
121
+ time = Time.parse("2012-10-26T08:45:42Z").to_i
122
+ driver.run do
123
+ driver.emit({"message" => "1st message"}, time)
124
+ end
125
+ assert_equal("/d/load?table=Logs",
126
+ @request_parser.request_url)
127
+ assert_equal([{"message" => "1st message"}],
128
+ JSON.parse(@request_body))
129
+ end
130
+
131
+ def test_multiple_messages
132
+ @response_body = JSON.generate([[0, 0.0, 0.0], [2]])
133
+ driver = create_driver("log")
134
+ time = Time.parse("2012-10-26T08:45:42Z").to_i
135
+ driver.run do
136
+ driver.emit({"message" => "1st message"}, time)
137
+ driver.emit({"message" => "2nd message"}, time + 1)
138
+ end
139
+ assert_equal("/d/load?table=Logs",
140
+ @request_parser.request_url)
141
+ assert_equal([
142
+ {"message" => "1st message"},
143
+ {"message" => "2nd message"},
144
+ ],
145
+ JSON.parse(@request_body))
106
146
  end
107
147
  end
108
148
  end
metadata CHANGED
@@ -1,164 +1,167 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-groonga
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-09-29 00:00:00.000000000 Z
11
+ date: 2014-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - '>='
17
+ - - ">="
18
18
  - !ruby/object:Gem::Version
19
19
  version: '0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - '>='
24
+ - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: '0'
27
27
  - !ruby/object:Gem::Dependency
28
- name: gqtp
28
+ name: groonga-client
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '>='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: 1.0.3
33
+ version: '0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 1.0.3
40
+ version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: groonga-command-parser
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - '>='
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '0'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - '>='
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - '>='
59
+ - - ">="
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - '>='
66
+ - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: bundler
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: packnga
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - '>='
87
+ - - ">="
88
88
  - !ruby/object:Gem::Version
89
89
  version: 0.9.6
90
90
  type: :development
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - '>='
94
+ - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: 0.9.6
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: test-unit
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - '>='
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '0'
104
104
  type: :development
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - '>='
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '0'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: test-unit-notify
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
- - - '>='
115
+ - - ">="
116
116
  - !ruby/object:Gem::Version
117
117
  version: '0'
118
118
  type: :development
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
- - - '>='
122
+ - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
125
  - !ruby/object:Gem::Dependency
126
126
  name: redcarpet
127
127
  requirement: !ruby/object:Gem::Requirement
128
128
  requirements:
129
- - - '>='
129
+ - - ">="
130
130
  - !ruby/object:Gem::Version
131
131
  version: '0'
132
132
  type: :development
133
133
  prerelease: false
134
134
  version_requirements: !ruby/object:Gem::Requirement
135
135
  requirements:
136
- - - '>='
136
+ - - ">="
137
137
  - !ruby/object:Gem::Version
138
138
  version: '0'
139
- description: Groonga users can replicate their data by fluent-plugin-groonga
139
+ description: There are two usages. 1) Store data into Groonga. 2) Implement Groonga
140
+ replication system. See documentation for details.
140
141
  email:
141
142
  - kou@clear-code.com
142
143
  executables: []
143
144
  extensions: []
144
145
  extra_rdoc_files: []
145
146
  files:
146
- - README.md
147
+ - ".yardopts"
147
148
  - Gemfile
149
+ - README.md
150
+ - doc/text/configuration.md
151
+ - doc/text/constitution.md
152
+ - doc/text/lgpl-2.1.txt
153
+ - doc/text/news.md
148
154
  - fluent-plugin-groonga.gemspec
149
- - .yardopts
150
- - lib/fluent/plugin/out_groonga.rb
151
155
  - lib/fluent/plugin/in_groonga.rb
156
+ - lib/fluent/plugin/out_groonga.rb
157
+ - sample/command.conf
152
158
  - sample/gqtp.conf
153
159
  - sample/http.conf
154
- - sample/command.conf
155
- - doc/text/configuration.md
156
- - doc/text/news.md
157
- - doc/text/constitution.md
158
- - doc/text/lgpl-2.1.txt
159
- - test/test_output.rb
160
+ - sample/store.conf
161
+ - test/output/test_type_guesser.rb
160
162
  - test/run-test.rb
161
163
  - test/test_input.rb
164
+ - test/test_output.rb
162
165
  homepage: https://github.com/groonga/fluent-plugin-groonga
163
166
  licenses:
164
167
  - LGPL-2.1
@@ -169,22 +172,24 @@ require_paths:
169
172
  - lib
170
173
  required_ruby_version: !ruby/object:Gem::Requirement
171
174
  requirements:
172
- - - '>='
175
+ - - ">="
173
176
  - !ruby/object:Gem::Version
174
177
  version: '0'
175
178
  required_rubygems_version: !ruby/object:Gem::Requirement
176
179
  requirements:
177
- - - '>='
180
+ - - ">="
178
181
  - !ruby/object:Gem::Version
179
182
  version: '0'
180
183
  requirements: []
181
184
  rubyforge_project:
182
- rubygems_version: 2.0.7
185
+ rubygems_version: 2.2.2
183
186
  signing_key:
184
187
  specification_version: 4
185
- summary: Fluentd plugin collection for groonga users
188
+ summary: Fluentd plugin to store data into Groonga and implement Groonga replication
189
+ system.
186
190
  test_files:
187
191
  - test/test_output.rb
188
- - test/run-test.rb
189
192
  - test/test_input.rb
193
+ - test/run-test.rb
194
+ - test/output/test_type_guesser.rb
190
195
  has_rdoc: