groonga-client 0.5.4 → 0.5.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 31002e6c2a6cc64ea8249019c9118dcc785e5de9
4
- data.tar.gz: '03379d77558768d720f2abf650fc5a2cca03e4d5'
3
+ metadata.gz: b31699984e9b4cfb9751dc8cf4481031d10a2fa1
4
+ data.tar.gz: ea990b190ba221e32990e5b2f08118acb3b2ca63
5
5
  SHA512:
6
- metadata.gz: ec7bef6d724d2601fa4b757522ba50439784d50b503ba670831b01cbd0068d0dd310cd30e3513902925d6558a340d6319f89e5576a72cba5722513de1c3fe5dc
7
- data.tar.gz: c466483d16efc8e6b1f66256bfba88faac81c11877552c9060d501b2c6d1f9befd5d71c1982833e9ea02e22152169ba80fb6fc0efcd973616d9b06586eae616f
6
+ metadata.gz: 3485123969cc7cdf43b7c1a8afba2ec7b58d5eff387c3506bcdc344bd0a6d4da3ddf5c65dca1018fe58fa94531510c5f47ba4b5e455ac988fbc44fd0b831631e
7
+ data.tar.gz: fc4ce6ee9206924e5b44d59243fb8a1544e15cf5cf7adcd87105c977443c4aa233e4165a99d3df01d1914841f3178a3c274779172d49449d835eb4fd1fc02dc0
data/bin/groonga-client CHANGED
@@ -17,12 +17,6 @@
17
17
  # License along with this library; if not, write to the Free Software
18
18
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
19
 
20
- require "net/http"
21
-
22
- if Net::HTTP.const_defined?(:IDEMPOTENT_METHODS_)
23
- Net::HTTP::IDEMPOTENT_METHODS_.clear
24
- end
25
-
26
20
  require "groonga/client/command-line/groonga-client"
27
21
 
28
22
  command_line = Groonga::Client::CommandLine::GroongaClient.new
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+ # -*- ruby -*-
3
+ #
4
+ # Copyright (C) 2015-2016 Kouhei Sutou <kou@clear-code.com>
5
+ # Copyright (C) 2017 Kentaro Hayashi <hayashi@clear-code.com>
6
+ #
7
+ # This library is free software; you can redistribute it and/or
8
+ # modify it under the terms of the GNU Lesser General Public
9
+ # License as published by the Free Software Foundation; either
10
+ # version 2.1 of the License, or (at your option) any later version.
11
+ #
12
+ # This library is distributed in the hope that it will be useful,
13
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15
+ # Lesser General Public License for more details.
16
+ #
17
+ # You should have received a copy of the GNU Lesser General Public
18
+ # License along with this library; if not, write to the Free Software
19
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
+
21
+ require "groonga/client/command-line/groonga-client-index-check"
22
+
23
+ command_line = Groonga::Client::CommandLine::GroongaClientIndexCheck.new
24
+ exit(command_line.run(ARGV))
data/doc/text/news.md CHANGED
@@ -1,5 +1,16 @@
1
1
  # NEWS
2
2
 
3
+ ## 0.5.5 - 2017-10-31
4
+
5
+ ### Improvements
6
+
7
+ * `groonga-client-index-check`: Added a new command that finds
8
+ indexes which doesn't have source or indexes whose content is
9
+ broken.
10
+
11
+ * Disabled auto retry feature by `net/http` explicitly because `GET`
12
+ isn't idempotent request in Groonga such as `delete` command.
13
+
3
14
  ## 0.5.4 - 2017-10-27
4
15
 
5
16
  ### Improvements
@@ -49,7 +49,7 @@ Gem::Specification.new do |spec|
49
49
 
50
50
  spec.add_runtime_dependency("gqtp", ">= 1.0.4")
51
51
  spec.add_runtime_dependency("groonga-command", ">= 1.2.8")
52
- spec.add_runtime_dependency("groonga-command-parser", ">= 1.0.7")
52
+ spec.add_runtime_dependency("groonga-command-parser", ">= 1.1.0")
53
53
  spec.add_runtime_dependency("hashie")
54
54
 
55
55
  spec.add_development_dependency("bundler")
@@ -0,0 +1,190 @@
1
+ # Copyright (C) 2015-2017 Kouhei Sutou <kou@clear-code.com>
2
+ # Copyright (C) 2017 Kentaro Hayashi <hayashi@clear-code.com>
3
+ #
4
+ # This library is free software; you can redistribute it and/or
5
+ # modify it under the terms of the GNU Lesser General Public
6
+ # License as published by the Free Software Foundation; either
7
+ # version 2.1 of the License, or (at your option) any later version.
8
+ #
9
+ # This library is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12
+ # Lesser General Public License for more details.
13
+ #
14
+ # You should have received a copy of the GNU Lesser General Public
15
+ # License along with this library; if not, write to the Free Software
16
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
+
18
+ require "groonga/client"
19
+ require "groonga/client/command-line/parser"
20
+ require "groonga/client/command-line/runner"
21
+
22
+ module Groonga
23
+ class Client
24
+ module CommandLine
25
+ class GroongaClientIndexCheck
26
+ def initialize
27
+ @available_methods = [:source, :content]
28
+ @methods = []
29
+ end
30
+
31
+ def run(arguments)
32
+ parser = Parser.new
33
+ target_names = parser.parse(arguments) do |option_parser|
34
+ parse_command_line(option_parser)
35
+ end
36
+
37
+ if @methods.empty?
38
+ @methods = @available_methods
39
+ end
40
+
41
+ parser.open_client do |client|
42
+ checker = Checker.new(client, @methods, target_names)
43
+ checker.run
44
+ end
45
+ end
46
+
47
+ private
48
+ def parse_command_line(parser)
49
+ parser.banner += " [LEXICON1.INDEX1 LEXICON2.INDEX2 LEXICON3 ...]"
50
+
51
+ parser.separator("")
52
+ parser.separator("If no indexes are specified, " +
53
+ "all indexes are checked.")
54
+
55
+ parser.separator("")
56
+ parser.separator("Method:")
57
+
58
+ parser.on("--method=METHOD", @available_methods,
59
+ "Specify a method how to check indexes.",
60
+ "You can specify this option multiple times",
61
+ "to use multiple methods in one execution.",
62
+ "All methods are used by default.",
63
+ "Available methods:",
64
+ " source: Find indexes that don't have source.",
65
+ " content: Find indexes whose content is broken.",
66
+ "(#{@available_methods.join(", ")})") do |method|
67
+ @methods << method
68
+ end
69
+ end
70
+
71
+ class Checker < Runner
72
+ def initialize(client, methods, target_names)
73
+ super(client)
74
+ @methods = methods
75
+ @target_names = target_names
76
+ end
77
+
78
+ private
79
+ def run_internal
80
+ succeeded = true
81
+ each_target_index_column do |index_column|
82
+ @methods.each do |method|
83
+ unless __send__("check_#{method}", index_column)
84
+ succeeded = false
85
+ end
86
+ end
87
+ end
88
+ succeeded
89
+ end
90
+
91
+ def each_target_index_column
92
+ table_list.each do |table|
93
+ next unless target_table?(table)
94
+ column_list(table.name).each do |column|
95
+ next unless column.index?
96
+ next unless target_column?(column)
97
+ yield(column)
98
+ end
99
+ end
100
+ end
101
+
102
+ def target_table?(table)
103
+ return true if @target_names.empty?
104
+ @target_names.any? do |name|
105
+ if name.include?(".")
106
+ index_table_name = name.split(".").first
107
+ index_table_name == table.name
108
+ else
109
+ name == table.name
110
+ end
111
+ end
112
+ end
113
+
114
+ def target_column?(column)
115
+ return true if @target_names.empty?
116
+ @target_names.any? do |name|
117
+ if name.include?(".")
118
+ name == column.full_name
119
+ else
120
+ name == column.domain
121
+ end
122
+ end
123
+ end
124
+
125
+ def check_source(column)
126
+ return true unless column.source.empty?
127
+ $stderr.puts("Source is missing: <#{column.full_name}>")
128
+ false
129
+ end
130
+
131
+ def valid_token?(source_table_name,
132
+ full_index_column_name1,
133
+ full_index_column_name2,
134
+ token)
135
+ case token
136
+ when String
137
+ value = Groonga::Client::ScriptSyntax.format_string(token)
138
+ else
139
+ value = token
140
+ end
141
+ response1 = select(source_table_name,
142
+ :filter => "#{full_index_column_name1} @ #{value}",
143
+ :output_columns => "_id",
144
+ :limit => "-1",
145
+ :sort_keys => "_id")
146
+ response2 = select(source_table_name,
147
+ :filter => "#{full_index_column_name2} @ #{value}",
148
+ :output_columns => "_id",
149
+ :limit => "-1",
150
+ :sort_keys => "_id")
151
+ response1.records == response2.records
152
+ end
153
+
154
+ def check_content(index_column)
155
+ return if index_column.source.empty?
156
+
157
+ lexicon_name = index_column.domain
158
+ index_column_name = index_column.name
159
+ suffix = Time.now.strftime("%Y%m%d%H%M%S_%N")
160
+ new_index_column_name = "#{index_column_name}_#{suffix}"
161
+ full_index_column_name = index_column.full_name
162
+ full_new_index_column_name = "#{full_index_column_name}_#{suffix}"
163
+ source_table = index_column.range
164
+ column_create_similar(lexicon_name,
165
+ new_index_column_name,
166
+ index_column_name)
167
+ begin
168
+ response = select(lexicon_name,
169
+ :limit => "-1",
170
+ :output_columns => "_key")
171
+ response.records.each do |record|
172
+ token = record["_key"]
173
+ unless valid_token?(source_table,
174
+ full_index_column_name,
175
+ full_new_index_column_name,
176
+ token)
177
+ $stderr.puts("Broken: #{index_column.full_name}: <#{token}>")
178
+ return false
179
+ end
180
+ end
181
+ true
182
+ ensure
183
+ column_remove(lexicon_name, new_index_column_name)
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -14,39 +14,31 @@
14
14
  # License along with this library; if not, write to the Free Software
15
15
  # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
16
 
17
- require "optparse"
18
17
  require "json"
19
18
 
20
19
  require "groonga/client"
20
+ require "groonga/client/command-line/parser"
21
+ require "groonga/client/command-line/runner"
21
22
 
22
23
  module Groonga
23
24
  class Client
24
25
  module CommandLine
25
26
  class GroongaClientIndexRecreate
26
27
  def initialize
27
- @url = nil
28
- @protocol = :http
29
- @host = "localhost"
30
- @port = nil
31
-
32
- @read_timeout = -1
33
-
34
28
  @interval = :day
35
29
 
36
30
  @n_workers = 0
37
31
  end
38
32
 
39
- def run(argv)
40
- target_indexes = parse_command_line(argv)
33
+ def run(arguments)
34
+ parser = Parser.new(:read_timeout => -1)
35
+ indexes = parser.parse(arguments) do |option_parser|
36
+ parse_command_line(option_parser)
37
+ end
41
38
 
42
- Client.open(:url => @url,
43
- :protocol => @protocol,
44
- :host => @host,
45
- :port => @port,
46
- :read_timeout => @read_timeout,
47
- :backend => :synchronous) do |client|
48
- runner = Runner.new(client, @interval, target_indexes)
49
- runner.run do
39
+ parser.open_client do |client|
40
+ recreator = Recreator.new(client, @interval, indexes)
41
+ recreator.run do
50
42
  @n_workers.times do
51
43
  client.database_unmap
52
44
  end
@@ -55,48 +47,9 @@ module Groonga
55
47
  end
56
48
 
57
49
  private
58
- def parse_command_line(argv)
59
- parser = OptionParser.new
60
- parser.version = VERSION
50
+ def parse_command_line(parser)
61
51
  parser.banner += " LEXICON1.INDEX1 LEXICON2.INDEX2 ..."
62
52
 
63
- parser.separator("")
64
- parser.separator("Connection:")
65
-
66
- parser.on("--url=URL",
67
- "URL to connect to Groonga server.",
68
- "If this option is specified,",
69
- "--protocol, --host and --port are ignored.") do |url|
70
- @url = url
71
- end
72
-
73
- available_protocols = [:http, :gqtp]
74
- parser.on("--protocol=PROTOCOL", [:http, :gqtp],
75
- "Protocol to connect to Groonga server.",
76
- "[#{available_protocols.join(", ")}]",
77
- "(#{@protocol})") do |protocol|
78
- @protocol = protocol
79
- end
80
-
81
- parser.on("--host=HOST",
82
- "Groonga server to be connected.",
83
- "(#{@host})") do |host|
84
- @host = host
85
- end
86
-
87
- parser.on("--port=PORT", Integer,
88
- "Port number of Groonga server to be connected.",
89
- "(auto)") do |port|
90
- @port = port
91
- end
92
-
93
- parser.on("--read-timeout=TIMEOUT", Integer,
94
- "Timeout on reading response from Groonga server.",
95
- "You can disable timeout by specifying -1.",
96
- "(#{@read_timeout})") do |timeout|
97
- @read_timeout = timeout
98
- end
99
-
100
53
  parser.separator("")
101
54
  parser.separator("Configuration:")
102
55
 
@@ -117,91 +70,25 @@ module Groonga
117
70
  "(#{@n_workers})") do |n|
118
71
  @n_workers = n
119
72
  end
120
-
121
- target_indexes = parser.parse(argv)
122
-
123
- @port ||= default_port(@protocol)
124
-
125
- target_indexes
126
- end
127
-
128
- def default_port(protocol)
129
- case protocol
130
- when :http
131
- 10041
132
- when :gqtp
133
- 10043
134
- end
135
73
  end
136
74
 
137
- class Runner
75
+ class Recreator < Runner
138
76
  def initialize(client, interval, target_indexes)
139
- @client = client
77
+ super(client)
140
78
  @interval = interval
141
79
  @target_indexes = target_indexes
142
80
  @now = Time.now
143
81
  end
144
82
 
145
- def run
146
- catch do |tag|
147
- @abort_tag = tag
148
- alias_column = ensure_alias_column
149
- @target_indexes.each do |index|
150
- current_index = recreate_index(index, alias_column)
151
- remove_old_indexes(index, current_index)
152
- end
153
- yield if block_given?
154
- true
155
- end
156
- end
157
-
158
83
  private
159
- def abort_run(message)
160
- $stderr.puts(message)
161
- throw(@abort_tag, false)
162
- end
163
-
164
- def execute_command(name, arguments={})
165
- response = @client.execute(name, arguments)
166
- unless response.success?
167
- abort_run("Failed to run #{name}: #{response.inspect}")
84
+ def run_internal
85
+ alias_column = ensure_alias_column
86
+ @target_indexes.each do |index|
87
+ current_index = recreate_index(index, alias_column)
88
+ remove_old_indexes(index, current_index)
168
89
  end
169
- response
170
- end
171
-
172
- def config_get(key)
173
- execute_command(:config_get, :key => key).body
174
- end
175
-
176
- def config_set(key, value)
177
- execute_command(:config_set, :key => key, :value => value).body
178
- end
179
-
180
- def object_exist?(name)
181
- execute_command(:object_exist, :name => name).body
182
- end
183
-
184
- def column_rename(table, name, new_name)
185
- execute_command(:column_rename,
186
- :table => table,
187
- :name => name,
188
- :new_name => new_name).body
189
- end
190
-
191
- def column_list(table)
192
- execute_command(:column_list, :table => table)
193
- end
194
-
195
- def column_remove(table, column)
196
- execute_command(:column_remove,
197
- :table => table,
198
- :name => column)
199
- end
200
-
201
- def column_create_similar(table, column_name, base_column_name)
202
- info = execute_command(:schema)["#{table}.#{base_column_name}"]
203
- arguments = info.command.arguments.merge("name" => column_name)
204
- execute_command(:column_create, arguments).body
90
+ yield if block_given?
91
+ true
205
92
  end
206
93
 
207
94
  def set_alias(alias_column, alias_name, real_name)
@@ -229,10 +116,9 @@ module Groonga
229
116
  def resolve_alias(alias_column, key)
230
117
  table, column = alias_column.split(".", 2)
231
118
  filter = "_key == #{ScriptSyntax.format_string(key)}"
232
- response = execute_command(:select,
233
- :table => table,
234
- :filter => filter,
235
- :output_columns => column)
119
+ response = select(table,
120
+ :filter => filter,
121
+ :output_columns => column)
236
122
  return nil if response.n_hits.zero?
237
123
  response.records.first[column]
238
124
  end