groonga-command-parser 1.1.2 → 1.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3f848ed21767fd2e9f3d30712ea34363175b3d02
4
- data.tar.gz: 158dd8b58703d631723e8f44b877ce3e9c69036f
3
+ metadata.gz: d15b5c7601c61b757b5f422f0edb1f8fb49f904a
4
+ data.tar.gz: 6bb40d256f229dbf2e99884facc76d6dc1da11f4
5
5
  SHA512:
6
- metadata.gz: 21cff9a6346dc77d4de8449f9f736ef029dea4df2fa18c51ea29d805a328de44d3b34bc9e7ddc626f0c65339d621e06ab0a5cecd8386829ccfca20f81f464365
7
- data.tar.gz: 2cd3f8e7f8a414b331710b2e6f521976c7aa3eb57431e29b44f654e1715fe2df94a2513bcd099fb5f03fcb5e7b7245b80f68f1740f752f9217b7339d3e970c61
6
+ metadata.gz: 618025a3101618ce4d4dde8074158c765faa4c179de98019a2375f66419e3849a5dcec40c59a9e5d0b3230ac2a875b1b6dda7247f2dd17ca7a4fb2f9f4d1cfcf
7
+ data.tar.gz: 6a5f6f60e9e002bdf64949af7dbe5fb6c229b864608f695256111d2d0e1da99792e48b2c560b0721e5bdd2943d93bc7b08b9621657c5d764dc19d5d90ac15888
@@ -0,0 +1,22 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # Copyright (C) 2019 Kouhei Sutou <kou@clear-code.com>
4
+ #
5
+ # This library is free software; you can redistribute it and/or
6
+ # modify it under the terms of the GNU Lesser General Public
7
+ # License as published by the Free Software Foundation; either
8
+ # version 2.1 of the License, or (at your option) any later version.
9
+ #
10
+ # This library is distributed in the hope that it will be useful,
11
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13
+ # Lesser General Public License for more details.
14
+ #
15
+ # You should have received a copy of the GNU Lesser General Public
16
+ # License along with this library; if not, write to the Free Software
17
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18
+
19
+ require "groonga/command/parser/command/groonga-command-filter"
20
+
21
+ command = Groonga::Command::Parser::Command::GroongaCommandFilter.new
22
+ exit(command.run(ARGV))
@@ -1,5 +1,19 @@
1
1
  # News
2
2
 
3
+ ## 1.1.3: 2019-03-22
4
+
5
+ ### Improvements
6
+
7
+ * Added a error check for grabage after load.
8
+ * `groonga-command-filter`: Added.
9
+ * `groonga-command-filter`: Added `--no-include-schema`.
10
+ * `groonga-command-filter`: Added `--no-include-load`.
11
+ * `groonga-command-convert-format`: Added new converter
12
+ for converting to the Elasticsearch format.
13
+ * `groonga-command-convert-format`: Added `--elasticsearch-version`.
14
+ * Required groonga-command 1.4.0 or later for converting to
15
+ the Elasticsearch format.
16
+
3
17
  ## 1.1.2: 2017-10-31
4
18
 
5
19
  ### Fixes
@@ -52,7 +52,7 @@ Gem::Specification.new do |spec|
52
52
  spec.licenses = ["LGPLv2.1+"]
53
53
  spec.require_paths = ["lib"]
54
54
 
55
- spec.add_runtime_dependency("groonga-command", ">= 1.3.2")
55
+ spec.add_runtime_dependency("groonga-command", ">= 1.4.0")
56
56
  spec.add_runtime_dependency("json-stream")
57
57
 
58
58
  spec.add_development_dependency("test-unit")
@@ -1,4 +1,4 @@
1
- # Copyright (C) 2011-2017 Kouhei Sutou <kou@clear-code.com>
1
+ # Copyright (C) 2011-2019 Kouhei Sutou <kou@clear-code.com>
2
2
  #
3
3
  # This library is free software; you can redistribute it and/or
4
4
  # modify it under the terms of the GNU Lesser General Public
@@ -117,7 +117,8 @@ module Groonga
117
117
  end
118
118
  end
119
119
 
120
- def initialize
120
+ def initialize(options={})
121
+ @need_original_source = options.fetch(:need_original_source, true)
121
122
  reset
122
123
  initialize_hooks
123
124
  end
@@ -134,9 +135,13 @@ module Groonga
134
135
  # command, this method raises {Parser::Error}.
135
136
  def finish
136
137
  if @loading
137
- raise Error.new("not completed",
138
- @command.original_source.lines.to_a.last,
139
- "")
138
+ original_source = @command.original_source
139
+ if original_source
140
+ last_line = original_source.lines.to_a.last
141
+ else
142
+ last_line = ""
143
+ end
144
+ raise Error.new("not completed", last_line, "")
140
145
  else
141
146
  catch do |tag|
142
147
  parse_line(@buffer)
@@ -245,7 +250,7 @@ module Groonga
245
250
  @command = parse_command(line)
246
251
  return if @command.nil?
247
252
 
248
- @command.original_source = line
253
+ @command.original_source = line if @need_original_source
249
254
  process_command
250
255
  end
251
256
  end
@@ -265,7 +270,7 @@ module Groonga
265
270
  on_load_complete(@command)
266
271
  reset
267
272
  else
268
- @command.original_source << "\n"
273
+ @command.original_source << "\n" if @need_original_source
269
274
  @loading = true
270
275
  initialize_load_values_parser
271
276
  end
@@ -316,6 +321,11 @@ module Groonga
316
321
  def parse_command_line(command_line)
317
322
  splitter = CommandLineSplitter.new(command_line)
318
323
  name, *arguments = splitter.split
324
+ if name.nil?
325
+ raise Error.new("invalid command name",
326
+ command_line,
327
+ "")
328
+ end
319
329
  pair_arguments = {}
320
330
  ordered_arguments = []
321
331
  until arguments.empty?
@@ -359,7 +369,7 @@ module Groonga
359
369
  end
360
370
  @load_values_parser.on_consumed = lambda do |consumed|
361
371
  if @loading
362
- @command.original_source << consumed
372
+ @command.original_source << consumed if @need_original_source
363
373
  if @buffer.bytesize == consumed.bytesize
364
374
  @buffer = "".force_encoding("ASCII-8BIT")
365
375
  else
@@ -27,6 +27,7 @@ module Groonga
27
27
  @format = :command
28
28
  @uri_prefix = "http://localhost:10041"
29
29
  @pretty_print = true
30
+ @elasticsearch_version = 5
30
31
  end
31
32
 
32
33
  def run(argv=ARGV)
@@ -57,7 +58,7 @@ module Groonga
57
58
  option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
58
59
  option_parser.version = VERSION
59
60
 
60
- formats = [:uri, :command]
61
+ formats = [:uri, :command, :elasticsearch]
61
62
  option_parser.on("--format=FORMAT", formats,
62
63
  "Convert to FORMAT",
63
64
  "Available formats #{formats.join(', ')}",
@@ -78,6 +79,17 @@ module Groonga
78
79
  @pretty_print = boolean
79
80
  end
80
81
 
82
+ option_parser.on("--elasticsearch-version=VERSION",
83
+ "Specify the Elasticsearch version",
84
+ "Because the Elasticsearch's import format" +
85
+ " differs depending on version",
86
+ "Currently, we can specify 5, 6, 7, and 8" +
87
+ " in this option",
88
+ "Available only in elasticsearch format",
89
+ "[#{@elasticsearch_version}]") do |version|
90
+ @elasticsearch_version = version
91
+ end
92
+
81
93
  option_parser.parse!(argv)
82
94
  end
83
95
 
@@ -96,6 +108,8 @@ module Groonga
96
108
  case @format
97
109
  when :uri
98
110
  "#{@uri_prefix}#{command.to_uri_format}"
111
+ when :elasticsearch
112
+ command.to_elasticsearch_format(:version => @elasticsearch_version)
99
113
  else
100
114
  command.to_command_format(:pretty_print => @pretty_print)
101
115
  end
@@ -0,0 +1,189 @@
1
+ # Copyright (C) 2019 Kouhei Sutou <kou@clear-code.com>
2
+ #
3
+ # This library is free software; you can redistribute it and/or
4
+ # modify it under the terms of the GNU Lesser General Public
5
+ # License as published by the Free Software Foundation; either
6
+ # version 2.1 of the License, or (at your option) any later version.
7
+ #
8
+ # This library is distributed in the hope that it will be useful,
9
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
10
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11
+ # Lesser General Public License for more details.
12
+ #
13
+ # You should have received a copy of the GNU Lesser General Public
14
+ # License along with this library; if not, write to the Free Software
15
+ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16
+
17
+ require "optparse"
18
+
19
+ require "groonga/command/parser"
20
+
21
+ module Groonga
22
+ module Command
23
+ class Parser
24
+ module Command
25
+ class GroongaCommandFilter
26
+ def initialize
27
+ @include_tables = {}
28
+ @include_schema = true
29
+ @include_load = true
30
+ end
31
+
32
+ def run(argv=ARGV)
33
+ begin
34
+ parse_options!(argv)
35
+ rescue OptionParser::ParseError
36
+ $stderr.puts($!.message)
37
+ return false
38
+ end
39
+
40
+ input_paths = argv
41
+ if input_paths.empty?
42
+ filter($stdin)
43
+ else
44
+ input_paths.each do |input_path|
45
+ File.open(input_path) do |input|
46
+ filter(input)
47
+ end
48
+ end
49
+ end
50
+
51
+ true
52
+ end
53
+
54
+ private
55
+ def parse_options!(argv)
56
+ option_parser = OptionParser.new
57
+ option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
58
+ option_parser.version = VERSION
59
+
60
+ option_parser.on("--include-column=TABLE.COLUMN",
61
+ "Remain only TABLE.COLUMN from load data.",
62
+ "You can specify multiple TABLE.COLUMN by",
63
+ "specifying this option multiple times.") do |table_column|
64
+ table, column = table_column.split(".", 2)
65
+ @include_tables[table] ||= {}
66
+ @include_tables[table][column] = true
67
+ end
68
+
69
+ option_parser.on("--no-include-schema",
70
+ "Remove schema manipulation commands") do |boolean|
71
+ @include_schema = boolean
72
+ end
73
+
74
+ option_parser.on("--no-include-load",
75
+ "Remove load command") do |boolean|
76
+ @include_load = boolean
77
+ end
78
+
79
+ option_parser.parse!(argv)
80
+ end
81
+
82
+ def filter(input)
83
+ parser = Parser.new(need_original_source: true)
84
+ parser.on_command do |command|
85
+ filter_command(command)
86
+ end
87
+ parser.on_load_start do |command|
88
+ filter_load_start(command)
89
+ end
90
+ parser.on_load_columns do |command, columns|
91
+ filter_load_columns(command, columns)
92
+ end
93
+ parser.on_load_value do |command, value|
94
+ filter_load_value(command, value)
95
+ end
96
+ parser.on_load_complete do |command|
97
+ filter_load_complete(command)
98
+ end
99
+ parser.on_comment do |comment|
100
+ puts("\##{comment}")
101
+ end
102
+ input.each_line do |line|
103
+ parser << line
104
+ end
105
+ parser.finish
106
+ end
107
+
108
+ private
109
+ def filter_command(command)
110
+ return unless @include_schema # TODO: Too loose
111
+ case command
112
+ when TableCreate
113
+ return unless target_table?(command.name)
114
+ puts(command)
115
+ when ColumnCreate
116
+ return unless target_column?(command.table, command.name)
117
+ puts(command)
118
+ else
119
+ puts(command)
120
+ end
121
+ end
122
+
123
+ def filter_load_start(command)
124
+ return unless @include_load
125
+ return unless target_table?(command.table)
126
+ puts(command)
127
+ puts("[")
128
+ @need_comma = false
129
+ end
130
+
131
+ def filter_load_columns(command, columns)
132
+ return unless @include_load
133
+ columns = extract_target_columns(command.table, columns)
134
+ return if columns.empty?
135
+ print(JSON.generate(columns))
136
+ @need_comma = true
137
+ end
138
+
139
+ def filter_load_value(command, value)
140
+ return unless @include_load
141
+ return unless target_table?(command.table)
142
+ value = extract_target_attributes(command.table,
143
+ command.columns,
144
+ value)
145
+ return if value.empty?
146
+ puts(",") if @need_comma
147
+ print(JSON.generate(value))
148
+ @need_comma = true
149
+ end
150
+
151
+ def filter_load_complete(command)
152
+ return unless @include_load
153
+ return unless target_table?(command.table)
154
+ puts("]")
155
+ end
156
+
157
+ def target_table?(table)
158
+ @include_tables.empty? or
159
+ @include_tables.key?(table)
160
+ end
161
+
162
+ def target_column?(table, column)
163
+ return true if @include_tables.empty?
164
+ columns = @include_tables[table]
165
+ return false if columns.nil?
166
+ column == "_key" or columns.key?(column)
167
+ end
168
+
169
+ def extract_target_columns(table, columns)
170
+ columns.find_all do |column|
171
+ target_column?(table, column)
172
+ end
173
+ end
174
+
175
+ def extract_target_attributes(table, columns, value)
176
+ case value
177
+ when ::Array
178
+ value.find_all.each_with_index do |_, i|
179
+ target_column?(table, columns[i])
180
+ end
181
+ when ::Hash
182
+ raise "TODO"
183
+ end
184
+ end
185
+ end
186
+ end
187
+ end
188
+ end
189
+ end
@@ -17,7 +17,7 @@
17
17
  module Groonga
18
18
  module Command
19
19
  class Parser
20
- VERSION = "1.1.2"
20
+ VERSION = "1.1.3"
21
21
  end
22
22
  end
23
23
  end
@@ -492,6 +492,21 @@ XXX
492
492
  JSON
493
493
  end
494
494
  end
495
+
496
+ def test_garbage_after_json
497
+ message = "invalid command name"
498
+ before = "\""
499
+ after = ""
500
+ error = Groonga::Command::Parser::Error.new(message, before, after)
501
+ assert_raise(error) do
502
+ @parser << <<-JSON
503
+ load --table Users
504
+ [
505
+ {"_key": "alice", "name": "Alice"}
506
+ ]"
507
+ JSON
508
+ end
509
+ end
495
510
  end
496
511
  end
497
512
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: groonga-command-parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2
4
+ version: 1.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kouhei Sutou
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-10-31 00:00:00.000000000 Z
11
+ date: 2019-03-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: groonga-command
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 1.3.2
19
+ version: 1.4.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
- version: 1.3.2
26
+ version: 1.4.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: json-stream
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -141,6 +141,7 @@ email:
141
141
  - kou@clear-code.com
142
142
  executables:
143
143
  - groonga-command-convert-format
144
+ - groonga-command-filter
144
145
  - groonga-command-logicalify
145
146
  extensions: []
146
147
  extra_rdoc_files: []
@@ -150,6 +151,7 @@ files:
150
151
  - README.md
151
152
  - Rakefile
152
153
  - bin/groonga-command-convert-format
154
+ - bin/groonga-command-filter
153
155
  - bin/groonga-command-logicalify
154
156
  - doc/text/lgpl-2.1.txt
155
157
  - doc/text/news.md
@@ -157,6 +159,7 @@ files:
157
159
  - lib/groonga/command/parser.rb
158
160
  - lib/groonga/command/parser/command-line-splitter.rb
159
161
  - lib/groonga/command/parser/command/groonga-command-convert-format.rb
162
+ - lib/groonga/command/parser/command/groonga-command-filter.rb
160
163
  - lib/groonga/command/parser/command/groonga-command-logicalify.rb
161
164
  - lib/groonga/command/parser/error.rb
162
165
  - lib/groonga/command/parser/load-values-parser.rb
@@ -192,8 +195,8 @@ specification_version: 4
192
195
  summary: Groonga-command-parser is a Ruby library to parses [groonga](http://groonga.org/)'s
193
196
  command syntax. You can write a program to process groonga's command by using groonga-command-parser.
194
197
  test_files:
198
+ - test/test-load-value-parser.rb
195
199
  - test/run-test.rb
196
200
  - test/test-parser.rb
197
- - test/test-load-value-parser.rb
198
- - test/test-command-line-splitter.rb
199
201
  - test/groonga-command-parser-test-utils.rb
202
+ - test/test-command-line-splitter.rb