groonga-command-parser 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f848ed21767fd2e9f3d30712ea34363175b3d02
-  data.tar.gz: 158dd8b58703d631723e8f44b877ce3e9c69036f
+  metadata.gz: d15b5c7601c61b757b5f422f0edb1f8fb49f904a
+  data.tar.gz: 6bb40d256f229dbf2e99884facc76d6dc1da11f4
 SHA512:
-  metadata.gz: 21cff9a6346dc77d4de8449f9f736ef029dea4df2fa18c51ea29d805a328de44d3b34bc9e7ddc626f0c65339d621e06ab0a5cecd8386829ccfca20f81f464365
-  data.tar.gz: 2cd3f8e7f8a414b331710b2e6f521976c7aa3eb57431e29b44f654e1715fe2df94a2513bcd099fb5f03fcb5e7b7245b80f68f1740f752f9217b7339d3e970c61
+  metadata.gz: 618025a3101618ce4d4dde8074158c765faa4c179de98019a2375f66419e3849a5dcec40c59a9e5d0b3230ac2a875b1b6dda7247f2dd17ca7a4fb2f9f4d1cfcf
+  data.tar.gz: 6a5f6f60e9e002bdf64949af7dbe5fb6c229b864608f695256111d2d0e1da99792e48b2c560b0721e5bdd2943d93bc7b08b9621657c5d764dc19d5d90ac15888

data/bin/groonga-command-filter

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "groonga/command/parser/command/groonga-command-filter"
+
+command = Groonga::Command::Parser::Command::GroongaCommandFilter.new
+exit(command.run(ARGV))

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # News
 
+## 1.1.3: 2019-03-22
+
+### Improvements
+
+  * Added a error check for grabage after load.
+  * `groonga-command-filter`: Added.
+  * `groonga-command-filter`: Added `--no-include-schema`.
+  * `groonga-command-filter`: Added `--no-include-load`.
+  * `groonga-command-convert-format`: Added new converter
+    for converting to the Elasticsearch format.
+  * `groonga-command-convert-format`: Added `--elasticsearch-version`.
+  * Required groonga-command 1.4.0 or later for converting to
+    the Elasticsearch format.
+
 ## 1.1.2: 2017-10-31
 
 ### Fixes

data/groonga-command-parser.gemspec

@@ -52,7 +52,7 @@ Gem::Specification.new do |spec|
   spec.licenses = ["LGPLv2.1+"]
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency("groonga-command", ">= 1.3.2")
+  spec.add_runtime_dependency("groonga-command", ">= 1.4.0")
   spec.add_runtime_dependency("json-stream")
 
   spec.add_development_dependency("test-unit")

data/lib/groonga/command/parser.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2017  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2011-2019  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -117,7 +117,8 @@ module Groonga
         end
       end
 
-      def initialize
+      def initialize(options={})
+        @need_original_source = options.fetch(:need_original_source, true)
         reset
         initialize_hooks
       end
@@ -134,9 +135,13 @@ module Groonga
       # command, this method raises {Parser::Error}.
       def finish
         if @loading
-          raise Error.new("not completed",
-                          @command.original_source.lines.to_a.last,
-                          "")
+          original_source = @command.original_source
+          if original_source
+            last_line = original_source.lines.to_a.last
+          else
+            last_line = ""
+          end
+          raise Error.new("not completed", last_line, "")
         else
           catch do |tag|
             parse_line(@buffer)
@@ -245,7 +250,7 @@ module Groonga
           @command = parse_command(line)
           return if @command.nil?
 
-          @command.original_source = line
+          @command.original_source = line if @need_original_source
           process_command
         end
       end
@@ -265,7 +270,7 @@ module Groonga
               on_load_complete(@command)
               reset
             else
-              @command.original_source << "\n"
+              @command.original_source << "\n" if @need_original_source
               @loading = true
               initialize_load_values_parser
             end
@@ -316,6 +321,11 @@ module Groonga
       def parse_command_line(command_line)
         splitter = CommandLineSplitter.new(command_line)
         name, *arguments = splitter.split
+        if name.nil?
+          raise Error.new("invalid command name",
+                          command_line,
+                          "")
+        end
         pair_arguments = {}
         ordered_arguments = []
         until arguments.empty?
@@ -359,7 +369,7 @@ module Groonga
         end
         @load_values_parser.on_consumed = lambda do |consumed|
           if @loading
-            @command.original_source << consumed
+            @command.original_source << consumed if @need_original_source
             if @buffer.bytesize == consumed.bytesize
               @buffer = "".force_encoding("ASCII-8BIT")
             else

data/lib/groonga/command/parser/command/groonga-command-convert-format.rb

@@ -27,6 +27,7 @@ module Groonga
             @format = :command
             @uri_prefix = "http://localhost:10041"
             @pretty_print = true
+            @elasticsearch_version = 5
           end
 
           def run(argv=ARGV)
@@ -57,7 +58,7 @@ module Groonga
             option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
             option_parser.version = VERSION
 
-            formats = [:uri, :command]
+            formats = [:uri, :command, :elasticsearch]
             option_parser.on("--format=FORMAT", formats,
                              "Convert to FORMAT",
                              "Available formats #{formats.join(', ')}",
@@ -78,6 +79,17 @@ module Groonga
               @pretty_print = boolean
             end
 
+            option_parser.on("--elasticsearch-version=VERSION",
+                             "Specify the Elasticsearch version",
+                             "Because the Elasticsearch's import format" +
+                                        " differs depending on version",
+                             "Currently, we can specify 5, 6, 7, and 8" +
+                                                      " in this option",
+                             "Available only in elasticsearch format",
+                             "[#{@elasticsearch_version}]") do |version|
+              @elasticsearch_version = version
+            end
+
             option_parser.parse!(argv)
           end
 
@@ -96,6 +108,8 @@ module Groonga
             case @format
             when :uri
               "#{@uri_prefix}#{command.to_uri_format}"
+            when :elasticsearch
+              command.to_elasticsearch_format(:version => @elasticsearch_version)
             else
               command.to_command_format(:pretty_print => @pretty_print)
             end

data/lib/groonga/command/parser/command/groonga-command-filter.rb

@@ -0,0 +1,189 @@
+# Copyright (C) 2019  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+require "groonga/command/parser"
+
+module Groonga
+  module Command
+    class Parser
+      module Command
+        class GroongaCommandFilter
+          def initialize
+            @include_tables = {}
+            @include_schema = true
+            @include_load = true
+          end
+
+          def run(argv=ARGV)
+            begin
+              parse_options!(argv)
+            rescue OptionParser::ParseError
+              $stderr.puts($!.message)
+              return false
+            end
+
+            input_paths = argv
+            if input_paths.empty?
+              filter($stdin)
+            else
+              input_paths.each do |input_path|
+                File.open(input_path) do |input|
+                  filter(input)
+                end
+              end
+            end
+
+            true
+          end
+
+          private
+          def parse_options!(argv)
+            option_parser = OptionParser.new
+            option_parser.banner += " INPUT_PATH1 INPUT_PATH2 ..."
+            option_parser.version = VERSION
+
+            option_parser.on("--include-column=TABLE.COLUMN",
+                             "Remain only TABLE.COLUMN from load data.",
+                             "You can specify multiple TABLE.COLUMN by",
+                             "specifying this option multiple times.") do |table_column|
+              table, column = table_column.split(".", 2)
+              @include_tables[table] ||= {}
+              @include_tables[table][column] = true
+            end
+
+            option_parser.on("--no-include-schema",
+                             "Remove schema manipulation commands") do |boolean|
+              @include_schema = boolean
+            end
+
+            option_parser.on("--no-include-load",
+                             "Remove load command") do |boolean|
+              @include_load = boolean
+            end
+
+            option_parser.parse!(argv)
+          end
+
+          def filter(input)
+            parser = Parser.new(need_original_source: true)
+            parser.on_command do |command|
+              filter_command(command)
+            end
+            parser.on_load_start do |command|
+              filter_load_start(command)
+            end
+            parser.on_load_columns do |command, columns|
+              filter_load_columns(command, columns)
+            end
+            parser.on_load_value do |command, value|
+              filter_load_value(command, value)
+            end
+            parser.on_load_complete do |command|
+              filter_load_complete(command)
+            end
+            parser.on_comment do |comment|
+              puts("\##{comment}")
+            end
+            input.each_line do |line|
+              parser << line
+            end
+            parser.finish
+          end
+
+          private
+          def filter_command(command)
+            return unless @include_schema # TODO: Too loose
+            case command
+            when TableCreate
+              return unless target_table?(command.name)
+              puts(command)
+            when ColumnCreate
+              return unless target_column?(command.table, command.name)
+              puts(command)
+            else
+              puts(command)
+            end
+          end
+
+          def filter_load_start(command)
+            return unless @include_load
+            return unless target_table?(command.table)
+            puts(command)
+            puts("[")
+            @need_comma = false
+          end
+
+          def filter_load_columns(command, columns)
+            return unless @include_load
+            columns = extract_target_columns(command.table, columns)
+            return if columns.empty?
+            print(JSON.generate(columns))
+            @need_comma = true
+          end
+
+          def filter_load_value(command, value)
+            return unless @include_load
+            return unless target_table?(command.table)
+            value = extract_target_attributes(command.table,
+                                              command.columns,
+                                              value)
+            return if value.empty?
+            puts(",") if @need_comma
+            print(JSON.generate(value))
+            @need_comma = true
+          end
+
+          def filter_load_complete(command)
+            return unless @include_load
+            return unless target_table?(command.table)
+            puts("]")
+          end
+
+          def target_table?(table)
+            @include_tables.empty? or
+              @include_tables.key?(table)
+          end
+
+          def target_column?(table, column)
+            return true if @include_tables.empty?
+            columns = @include_tables[table]
+            return false if columns.nil?
+            column == "_key" or columns.key?(column)
+          end
+
+          def extract_target_columns(table, columns)
+            columns.find_all do |column|
+              target_column?(table, column)
+            end
+          end
+
+          def extract_target_attributes(table, columns, value)
+            case value
+            when ::Array
+              value.find_all.each_with_index do |_, i|
+                target_column?(table, columns[i])
+              end
+            when ::Hash
+              raise "TODO"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/groonga/command/parser/version.rb

@@ -17,7 +17,7 @@
 module Groonga
   module Command
     class Parser
-      VERSION = "1.1.2"
+      VERSION = "1.1.3"
     end
   end
 end

data/test/test-parser.rb

@@ -492,6 +492,21 @@ XXX
               JSON
             end
           end
+
+          def test_garbage_after_json
+            message = "invalid command name"
+            before = "\""
+            after = ""
+            error = Groonga::Command::Parser::Error.new(message, before, after)
+            assert_raise(error) do
+              @parser << <<-JSON
+load --table Users
+[
+{"_key": "alice", "name": "Alice"}
+]"
+              JSON
+            end
+          end
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: groonga-command-parser
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-31 00:00:00.000000000 Z
+date: 2019-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: groonga-command
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.2
+        version: 1.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.3.2
+        version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: json-stream
   requirement: !ruby/object:Gem::Requirement
@@ -141,6 +141,7 @@ email:
 - kou@clear-code.com
 executables:
 - groonga-command-convert-format
+- groonga-command-filter
 - groonga-command-logicalify
 extensions: []
 extra_rdoc_files: []
@@ -150,6 +151,7 @@ files:
 - README.md
 - Rakefile
 - bin/groonga-command-convert-format
+- bin/groonga-command-filter
 - bin/groonga-command-logicalify
 - doc/text/lgpl-2.1.txt
 - doc/text/news.md
@@ -157,6 +159,7 @@ files:
 - lib/groonga/command/parser.rb
 - lib/groonga/command/parser/command-line-splitter.rb
 - lib/groonga/command/parser/command/groonga-command-convert-format.rb
+- lib/groonga/command/parser/command/groonga-command-filter.rb
 - lib/groonga/command/parser/command/groonga-command-logicalify.rb
 - lib/groonga/command/parser/error.rb
 - lib/groonga/command/parser/load-values-parser.rb
@@ -192,8 +195,8 @@ specification_version: 4
 summary: Groonga-command-parser is a Ruby library to parses [groonga](http://groonga.org/)'s
   command syntax. You can write a program to process groonga's command by using groonga-command-parser.
 test_files:
+- test/test-load-value-parser.rb
 - test/run-test.rb
 - test/test-parser.rb
-- test/test-load-value-parser.rb
-- test/test-command-line-splitter.rb
 - test/groonga-command-parser-test-utils.rb
+- test/test-command-line-splitter.rb