groonga-command-parser 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/groonga-command-filter +22 -0
- data/doc/text/news.md +14 -0
- data/groonga-command-parser.gemspec +1 -1
- data/lib/groonga/command/parser.rb +18 -8
- data/lib/groonga/command/parser/command/groonga-command-convert-format.rb +15 -1
- data/lib/groonga/command/parser/command/groonga-command-filter.rb +189 -0
- data/lib/groonga/command/parser/version.rb +1 -1
- data/test/test-parser.rb +15 -0
- metadata +9 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d15b5c7601c61b757b5f422f0edb1f8fb49f904a
|
4
|
+
data.tar.gz: 6bb40d256f229dbf2e99884facc76d6dc1da11f4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 618025a3101618ce4d4dde8074158c765faa4c179de98019a2375f66419e3849a5dcec40c59a9e5d0b3230ac2a875b1b6dda7247f2dd17ca7a4fb2f9f4d1cfcf
|
7
|
+
data.tar.gz: 6a5f6f60e9e002bdf64949af7dbe5fb6c229b864608f695256111d2d0e1da99792e48b2c560b0721e5bdd2943d93bc7b08b9621657c5d764dc19d5d90ac15888
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (C) 2019 Kouhei Sutou <kou@clear-code.com>
|
4
|
+
#
|
5
|
+
# This library is free software; you can redistribute it and/or
|
6
|
+
# modify it under the terms of the GNU Lesser General Public
|
7
|
+
# License as published by the Free Software Foundation; either
|
8
|
+
# version 2.1 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# This library is distributed in the hope that it will be useful,
|
11
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
12
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
13
|
+
# Lesser General Public License for more details.
|
14
|
+
#
|
15
|
+
# You should have received a copy of the GNU Lesser General Public
|
16
|
+
# License along with this library; if not, write to the Free Software
|
17
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
18
|
+
|
19
|
+
require "groonga/command/parser/command/groonga-command-filter"
|
20
|
+
|
21
|
+
command = Groonga::Command::Parser::Command::GroongaCommandFilter.new
|
22
|
+
exit(command.run(ARGV))
|
data/doc/text/news.md
CHANGED
@@ -1,5 +1,19 @@
|
|
1
1
|
# News
|
2
2
|
|
3
|
+
## 1.1.3: 2019-03-22
|
4
|
+
|
5
|
+
### Improvements
|
6
|
+
|
7
|
+
* Added a error check for grabage after load.
|
8
|
+
* `groonga-command-filter`: Added.
|
9
|
+
* `groonga-command-filter`: Added `--no-include-schema`.
|
10
|
+
* `groonga-command-filter`: Added `--no-include-load`.
|
11
|
+
* `groonga-command-convert-format`: Added new converter
|
12
|
+
for converting to the Elasticsearch format.
|
13
|
+
* `groonga-command-convert-format`: Added `--elasticsearch-version`.
|
14
|
+
* Required groonga-command 1.4.0 or later for converting to
|
15
|
+
the Elasticsearch format.
|
16
|
+
|
3
17
|
## 1.1.2: 2017-10-31
|
4
18
|
|
5
19
|
### Fixes
|
@@ -52,7 +52,7 @@ Gem::Specification.new do |spec|
|
|
52
52
|
spec.licenses = ["LGPLv2.1+"]
|
53
53
|
spec.require_paths = ["lib"]
|
54
54
|
|
55
|
-
spec.add_runtime_dependency("groonga-command", ">= 1.
|
55
|
+
spec.add_runtime_dependency("groonga-command", ">= 1.4.0")
|
56
56
|
spec.add_runtime_dependency("json-stream")
|
57
57
|
|
58
58
|
spec.add_development_dependency("test-unit")
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright (C) 2011-
|
1
|
+
# Copyright (C) 2011-2019 Kouhei Sutou <kou@clear-code.com>
|
2
2
|
#
|
3
3
|
# This library is free software; you can redistribute it and/or
|
4
4
|
# modify it under the terms of the GNU Lesser General Public
|
@@ -117,7 +117,8 @@ module Groonga
|
|
117
117
|
end
|
118
118
|
end
|
119
119
|
|
120
|
-
def initialize
|
120
|
+
def initialize(options={})
|
121
|
+
@need_original_source = options.fetch(:need_original_source, true)
|
121
122
|
reset
|
122
123
|
initialize_hooks
|
123
124
|
end
|
@@ -134,9 +135,13 @@ module Groonga
|
|
134
135
|
# command, this method raises {Parser::Error}.
|
135
136
|
def finish
|
136
137
|
if @loading
|
137
|
-
|
138
|
-
|
139
|
-
|
138
|
+
original_source = @command.original_source
|
139
|
+
if original_source
|
140
|
+
last_line = original_source.lines.to_a.last
|
141
|
+
else
|
142
|
+
last_line = ""
|
143
|
+
end
|
144
|
+
raise Error.new("not completed", last_line, "")
|
140
145
|
else
|
141
146
|
catch do |tag|
|
142
147
|
parse_line(@buffer)
|
@@ -245,7 +250,7 @@ module Groonga
|
|
245
250
|
@command = parse_command(line)
|
246
251
|
return if @command.nil?
|
247
252
|
|
248
|
-
@command.original_source = line
|
253
|
+
@command.original_source = line if @need_original_source
|
249
254
|
process_command
|
250
255
|
end
|
251
256
|
end
|
@@ -265,7 +270,7 @@ module Groonga
|
|
265
270
|
on_load_complete(@command)
|
266
271
|
reset
|
267
272
|
else
|
268
|
-
@command.original_source << "\n"
|
273
|
+
@command.original_source << "\n" if @need_original_source
|
269
274
|
@loading = true
|
270
275
|
initialize_load_values_parser
|
271
276
|
end
|
@@ -316,6 +321,11 @@ module Groonga
|
|
316
321
|
def parse_command_line(command_line)
|
317
322
|
splitter = CommandLineSplitter.new(command_line)
|
318
323
|
name, *arguments = splitter.split
|
324
|
+
if name.nil?
|
325
|
+
raise Error.new("invalid command name",
|
326
|
+
command_line,
|
327
|
+
"")
|
328
|
+
end
|
319
329
|
pair_arguments = {}
|
320
330
|
ordered_arguments = []
|
321
331
|
until arguments.empty?
|
@@ -359,7 +369,7 @@ module Groonga
|
|
359
369
|
end
|
360
370
|
@load_values_parser.on_consumed = lambda do |consumed|
|
361
371
|
if @loading
|
362
|
-
@command.original_source << consumed
|
372
|
+
@command.original_source << consumed if @need_original_source
|
363
373
|
if @buffer.bytesize == consumed.bytesize
|
364
374
|
@buffer = "".force_encoding("ASCII-8BIT")
|
365
375
|
else
|
@@ -27,6 +27,7 @@ module Groonga
|
|
27
27
|
@format = :command
|
28
28
|
@uri_prefix = "http://localhost:10041"
|
29
29
|
@pretty_print = true
|
30
|
+
@elasticsearch_version = 5
|
30
31
|
end
|
31
32
|
|
32
33
|
def run(argv=ARGV)
|
@@ -57,7 +58,7 @@ module Groonga
|
|
57
58
|
option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
|
58
59
|
option_parser.version = VERSION
|
59
60
|
|
60
|
-
formats = [:uri, :command]
|
61
|
+
formats = [:uri, :command, :elasticsearch]
|
61
62
|
option_parser.on("--format=FORMAT", formats,
|
62
63
|
"Convert to FORMAT",
|
63
64
|
"Available formats #{formats.join(', ')}",
|
@@ -78,6 +79,17 @@ module Groonga
|
|
78
79
|
@pretty_print = boolean
|
79
80
|
end
|
80
81
|
|
82
|
+
option_parser.on("--elasticsearch-version=VERSION",
|
83
|
+
"Specify the Elasticsearch version",
|
84
|
+
"Because the Elasticsearch's import format" +
|
85
|
+
" differs depending on version",
|
86
|
+
"Currently, we can specify 5, 6, 7, and 8" +
|
87
|
+
" in this option",
|
88
|
+
"Available only in elasticsearch format",
|
89
|
+
"[#{@elasticsearch_version}]") do |version|
|
90
|
+
@elasticsearch_version = version
|
91
|
+
end
|
92
|
+
|
81
93
|
option_parser.parse!(argv)
|
82
94
|
end
|
83
95
|
|
@@ -96,6 +108,8 @@ module Groonga
|
|
96
108
|
case @format
|
97
109
|
when :uri
|
98
110
|
"#{@uri_prefix}#{command.to_uri_format}"
|
111
|
+
when :elasticsearch
|
112
|
+
command.to_elasticsearch_format(:version => @elasticsearch_version)
|
99
113
|
else
|
100
114
|
command.to_command_format(:pretty_print => @pretty_print)
|
101
115
|
end
|
@@ -0,0 +1,189 @@
|
|
1
|
+
# Copyright (C) 2019 Kouhei Sutou <kou@clear-code.com>
|
2
|
+
#
|
3
|
+
# This library is free software; you can redistribute it and/or
|
4
|
+
# modify it under the terms of the GNU Lesser General Public
|
5
|
+
# License as published by the Free Software Foundation; either
|
6
|
+
# version 2.1 of the License, or (at your option) any later version.
|
7
|
+
#
|
8
|
+
# This library is distributed in the hope that it will be useful,
|
9
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
10
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
11
|
+
# Lesser General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU Lesser General Public
|
14
|
+
# License along with this library; if not, write to the Free Software
|
15
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
16
|
+
|
17
|
+
require "optparse"
|
18
|
+
|
19
|
+
require "groonga/command/parser"
|
20
|
+
|
21
|
+
module Groonga
|
22
|
+
module Command
|
23
|
+
class Parser
|
24
|
+
module Command
|
25
|
+
class GroongaCommandFilter
|
26
|
+
def initialize
|
27
|
+
@include_tables = {}
|
28
|
+
@include_schema = true
|
29
|
+
@include_load = true
|
30
|
+
end
|
31
|
+
|
32
|
+
def run(argv=ARGV)
|
33
|
+
begin
|
34
|
+
parse_options!(argv)
|
35
|
+
rescue OptionParser::ParseError
|
36
|
+
$stderr.puts($!.message)
|
37
|
+
return false
|
38
|
+
end
|
39
|
+
|
40
|
+
input_paths = argv
|
41
|
+
if input_paths.empty?
|
42
|
+
filter($stdin)
|
43
|
+
else
|
44
|
+
input_paths.each do |input_path|
|
45
|
+
File.open(input_path) do |input|
|
46
|
+
filter(input)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
true
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
def parse_options!(argv)
|
56
|
+
option_parser = OptionParser.new
|
57
|
+
option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
|
58
|
+
option_parser.version = VERSION
|
59
|
+
|
60
|
+
option_parser.on("--include-column=TABLE.COLUMN",
|
61
|
+
"Remain only TABLE.COLUMN from load data.",
|
62
|
+
"You can specify multiple TABLE.COLUMN by",
|
63
|
+
"specifying this option multiple times.") do |table_column|
|
64
|
+
table, column = table_column.split(".", 2)
|
65
|
+
@include_tables[table] ||= {}
|
66
|
+
@include_tables[table][column] = true
|
67
|
+
end
|
68
|
+
|
69
|
+
option_parser.on("--no-include-schema",
|
70
|
+
"Remove schema manipulation commands") do |boolean|
|
71
|
+
@include_schema = boolean
|
72
|
+
end
|
73
|
+
|
74
|
+
option_parser.on("--no-include-load",
|
75
|
+
"Remove load command") do |boolean|
|
76
|
+
@include_load = boolean
|
77
|
+
end
|
78
|
+
|
79
|
+
option_parser.parse!(argv)
|
80
|
+
end
|
81
|
+
|
82
|
+
def filter(input)
|
83
|
+
parser = Parser.new(need_original_source: true)
|
84
|
+
parser.on_command do |command|
|
85
|
+
filter_command(command)
|
86
|
+
end
|
87
|
+
parser.on_load_start do |command|
|
88
|
+
filter_load_start(command)
|
89
|
+
end
|
90
|
+
parser.on_load_columns do |command, columns|
|
91
|
+
filter_load_columns(command, columns)
|
92
|
+
end
|
93
|
+
parser.on_load_value do |command, value|
|
94
|
+
filter_load_value(command, value)
|
95
|
+
end
|
96
|
+
parser.on_load_complete do |command|
|
97
|
+
filter_load_complete(command)
|
98
|
+
end
|
99
|
+
parser.on_comment do |comment|
|
100
|
+
puts("\##{comment}")
|
101
|
+
end
|
102
|
+
input.each_line do |line|
|
103
|
+
parser << line
|
104
|
+
end
|
105
|
+
parser.finish
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
def filter_command(command)
|
110
|
+
return unless @include_schema # TODO: Too loose
|
111
|
+
case command
|
112
|
+
when TableCreate
|
113
|
+
return unless target_table?(command.name)
|
114
|
+
puts(command)
|
115
|
+
when ColumnCreate
|
116
|
+
return unless target_column?(command.table, command.name)
|
117
|
+
puts(command)
|
118
|
+
else
|
119
|
+
puts(command)
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def filter_load_start(command)
|
124
|
+
return unless @include_load
|
125
|
+
return unless target_table?(command.table)
|
126
|
+
puts(command)
|
127
|
+
puts("[")
|
128
|
+
@need_comma = false
|
129
|
+
end
|
130
|
+
|
131
|
+
def filter_load_columns(command, columns)
|
132
|
+
return unless @include_load
|
133
|
+
columns = extract_target_columns(command.table, columns)
|
134
|
+
return if columns.empty?
|
135
|
+
print(JSON.generate(columns))
|
136
|
+
@need_comma = true
|
137
|
+
end
|
138
|
+
|
139
|
+
def filter_load_value(command, value)
|
140
|
+
return unless @include_load
|
141
|
+
return unless target_table?(command.table)
|
142
|
+
value = extract_target_attributes(command.table,
|
143
|
+
command.columns,
|
144
|
+
value)
|
145
|
+
return if value.empty?
|
146
|
+
puts(",") if @need_comma
|
147
|
+
print(JSON.generate(value))
|
148
|
+
@need_comma = true
|
149
|
+
end
|
150
|
+
|
151
|
+
def filter_load_complete(command)
|
152
|
+
return unless @include_load
|
153
|
+
return unless target_table?(command.table)
|
154
|
+
puts("]")
|
155
|
+
end
|
156
|
+
|
157
|
+
def target_table?(table)
|
158
|
+
@include_tables.empty? or
|
159
|
+
@include_tables.key?(table)
|
160
|
+
end
|
161
|
+
|
162
|
+
def target_column?(table, column)
|
163
|
+
return true if @include_tables.empty?
|
164
|
+
columns = @include_tables[table]
|
165
|
+
return false if columns.nil?
|
166
|
+
column == "_key" or columns.key?(column)
|
167
|
+
end
|
168
|
+
|
169
|
+
def extract_target_columns(table, columns)
|
170
|
+
columns.find_all do |column|
|
171
|
+
target_column?(table, column)
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def extract_target_attributes(table, columns, value)
|
176
|
+
case value
|
177
|
+
when ::Array
|
178
|
+
value.find_all.each_with_index do |_, i|
|
179
|
+
target_column?(table, columns[i])
|
180
|
+
end
|
181
|
+
when ::Hash
|
182
|
+
raise "TODO"
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
data/test/test-parser.rb
CHANGED
@@ -492,6 +492,21 @@ XXX
|
|
492
492
|
JSON
|
493
493
|
end
|
494
494
|
end
|
495
|
+
|
496
|
+
def test_garbage_after_json
|
497
|
+
message = "invalid command name"
|
498
|
+
before = "\""
|
499
|
+
after = ""
|
500
|
+
error = Groonga::Command::Parser::Error.new(message, before, after)
|
501
|
+
assert_raise(error) do
|
502
|
+
@parser << <<-JSON
|
503
|
+
load --table Users
|
504
|
+
[
|
505
|
+
{"_key": "alice", "name": "Alice"}
|
506
|
+
]"
|
507
|
+
JSON
|
508
|
+
end
|
509
|
+
end
|
495
510
|
end
|
496
511
|
end
|
497
512
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: groonga-command-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kouhei Sutou
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-03-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: groonga-command
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
19
|
+
version: 1.4.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.
|
26
|
+
version: 1.4.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: json-stream
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -141,6 +141,7 @@ email:
|
|
141
141
|
- kou@clear-code.com
|
142
142
|
executables:
|
143
143
|
- groonga-command-convert-format
|
144
|
+
- groonga-command-filter
|
144
145
|
- groonga-command-logicalify
|
145
146
|
extensions: []
|
146
147
|
extra_rdoc_files: []
|
@@ -150,6 +151,7 @@ files:
|
|
150
151
|
- README.md
|
151
152
|
- Rakefile
|
152
153
|
- bin/groonga-command-convert-format
|
154
|
+
- bin/groonga-command-filter
|
153
155
|
- bin/groonga-command-logicalify
|
154
156
|
- doc/text/lgpl-2.1.txt
|
155
157
|
- doc/text/news.md
|
@@ -157,6 +159,7 @@ files:
|
|
157
159
|
- lib/groonga/command/parser.rb
|
158
160
|
- lib/groonga/command/parser/command-line-splitter.rb
|
159
161
|
- lib/groonga/command/parser/command/groonga-command-convert-format.rb
|
162
|
+
- lib/groonga/command/parser/command/groonga-command-filter.rb
|
160
163
|
- lib/groonga/command/parser/command/groonga-command-logicalify.rb
|
161
164
|
- lib/groonga/command/parser/error.rb
|
162
165
|
- lib/groonga/command/parser/load-values-parser.rb
|
@@ -192,8 +195,8 @@ specification_version: 4
|
|
192
195
|
summary: Groonga-command-parser is a Ruby library to parses [groonga](http://groonga.org/)'s
|
193
196
|
command syntax. You can write a program to process groonga's command by using groonga-command-parser.
|
194
197
|
test_files:
|
198
|
+
- test/test-load-value-parser.rb
|
195
199
|
- test/run-test.rb
|
196
200
|
- test/test-parser.rb
|
197
|
-
- test/test-load-value-parser.rb
|
198
|
-
- test/test-command-line-splitter.rb
|
199
201
|
- test/groonga-command-parser-test-utils.rb
|
202
|
+
- test/test-command-line-splitter.rb
|