grn2drn 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2e0a8d7c5929c03621ad56721c0cd286ce9f4496
-  data.tar.gz: 92308316da5be5c531c9620f7e6c19aec7b1481a
+  metadata.gz: 4940edafe622781d134c34040b00f8334c10dd49
+  data.tar.gz: ee1a8d7a39e90e1f7eea163233c6290aded4ac90
 SHA512:
-  metadata.gz: 205e98ee3714af39439967bf13ac5c6e554435cba3f6c03930525b5befeb38f1768f164ea3d6f488a4fa2c90bbce937ef2969c84d149c88800134c0b1c4b1f40
-  data.tar.gz: bed2c4438e9a1d00414f43bdc55c18ca97718bb7c697fba37c41131e1be50f97a1f76ae6099a7c3ff8a2d325715ca7720d1ebee5c3ab6b8f6cb405a80b17b5cd
+  metadata.gz: b90c36e609d65ae8555f7176fb38a6236f649ffdb5232e788abf3b5bca542dff3f1ec337e545e7fb89a46ba5ec54256c5074bfae615437c08b7f40c92989c5b4
+  data.tar.gz: 57500ec0df6177e1399641d305deb8903138092c2e17f4d54950b6d71628bd1273e83d1758e27d26af7d618ce2de0a0c940f2eff4b7680948b8cab4ed5c5d005

data/README.md

@@ -16,10 +16,27 @@ Grn2drn is a command to convert a *.grn dump file to a JSONs file for Droonga.
 
 ## Basic usage
 
+For migrating Groonga to Droonga:
+
+```
+% grndump /path/to/groonga/database > dump.grn
+% grn2drn --dataset Droonga dump.grn > droonga.jsons
+```
+
+You can send the converted data by `droonga-request` or `droonga-send`
+command. They are included in
+[droonga-client gem](http://rubygems.org/gems/droonga-client).
+
+
+For creating [catalog.json](http://droonga.org/reference/catalog/):
+
 ```
-% grn2drn --dataset Droonga /path/to/grn/dump/file
+% grndump --no-dump-tables /path/to/groonga/database > schema.grn
+% grn2drn-schema schema.grn > schema.json
 ```
 
+It generates JSON that can be embedded into catalog.json.
+
 ## Mailing list
 
 * English: [groonga-talk@lists.sourceforge.net](https://lists.sourceforge.net/lists/listinfo/groonga-talk)

data/bin/grn2drn

@@ -19,7 +19,7 @@
 require "ostruct"
 require "optparse"
 
-require "grn2drn/converter"
+require "grn2drn/command-converter"
 
 options = OpenStruct.new
 option_parser = OptionParser.new do |parser|
@@ -64,28 +64,35 @@ convert_options = {
   :reply_to => options.reply_to,
   :dataset => options.dataset,
 }
-converter = Grn2Drn::Converter.new(convert_options)
+converter = Grn2Drn::CommandConverter.new(convert_options)
 
 source_file = args[0]
 result_file = args[1]
 
-input = nil
-if source_file.nil?
-  input = STDIN.read
-else
-  input = File.read(source_file)
+def open_input(source_file)
+  if source_file.nil?
+    yield($stdin)
+  else
+    File.open(source_file) do |input|
+      yield(input)
+    end
+  end
 end
 
-result_file = args[1]
-
-if result_file.nil?
-  converter.convert(input) do |command|
-    puts(JSON.generate(command))
+def open_output(result_file)
+  if result_file.nil?
+    yield($stdout)
+  else
+    File.open("w", result_file) do |output|
+      yield(output)
+    end
   end
-else
-  File.open("w", result_file) do |file|
+end
+
+open_input(source_file) do |input|
+  open_output(result_file) do |output|
     converter.convert(input) do |command|
-      file.puts(JSON.generate(command))
+      output.puts(JSON.generate(command))
     end
   end
 end

data/bin/grn2drn-schema

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "json"
+
+require "grn2drn/schema-converter"
+
+converter = Grn2Drn::SchemaConverter.new
+begin
+  schema = converter.convert(ARGF)
+rescue Grn2Drn::Error
+  puts($!.message)
+  exit(false)
+end
+puts(JSON.pretty_generate(schema))

data/lib/grn2drn/{converter.rb → command-converter.rb}

@@ -22,7 +22,7 @@ require "json"
 require "groonga/command/parser"
 
 module Grn2Drn
-  class Converter
+  class CommandConverter
     def initialize(options={})
       @options = options
       @count = 0
@@ -44,22 +44,13 @@ module Grn2Drn
         end
       end
 
-      parsed_values = nil
       parsed_columns = nil
-      @command_parser.on_load_start do |command|
-        parsed_values = []
-        parsed_columns = nil
-      end
       @command_parser.on_load_columns do |command, columns|
         parsed_columns = columns
       end
       @command_parser.on_load_value do |command, value|
-        parsed_values << value
-      end
-      @command_parser.on_load_complete do |command|
-        command[:columns] ||= parsed_columns.join(",") if parsed_columns
-        command[:values] = parsed_values.to_json
-        split_load_command_to_add_commands(command, &block)
+        yield create_add_command(command, parsed_columns, value)
+        command.original_source.clear
       end
 
       input.each_line do |line|
@@ -123,34 +114,30 @@ module Grn2Drn
       create_message("column_create", command_to_body(command))
     end
 
-    def split_load_command_to_add_commands(command, &block)
-      columns = command.columns
-      values = command[:values]
-      values = JSON.parse(values)
-      values.each do |record|
-        body = {
-          "table" => command[:table],
-        }
-
-        if record.is_a?(Hash)
-          record = record.dup
-          body["key"] = record.delete("_key")
-          record_values = record
-        else
-          record_values = {}
-          record.each_with_index do |value, column_index|
-            column = columns[column_index]
-            if column == "_key"
-              body["key"] = value
-            else
-              record_values[column] = value
-            end
+    def create_add_command(command, columns, record)
+      table = command[:table]
+      body = {
+        "table" => table,
+      }
+
+      if record.is_a?(Hash)
+        values = record.dup
+        body["key"] = values.delete("_key")
+        body["values"] = values
+      else
+        values = {}
+        record.each_with_index do |value, column_index|
+          column = columns[column_index]
+          if column == "_key"
+            body["key"] = value
+          else
+            values[column] = value
           end
         end
-        body["values"] = record_values unless record_values.empty?
-
-        yield create_message("add", body)
+        body["values"] = values
       end
+
+      create_message("add", body)
     end
 
     def create_select_command(command)

data/lib/grn2drn/error.rb

@@ -0,0 +1,19 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+module Grn2Drn
+  class Error < StandardError
+  end
+end

data/lib/grn2drn/schema-converter.rb

@@ -0,0 +1,207 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "groonga/command/parser"
+
+require "grn2drn/error"
+
+module Grn2Drn
+  class SchemaConverter
+    class UnknownTable < Error
+      attr_reader :table
+      def initialize(table)
+        @table = table
+        super("Unknown table: <#{@table}>")
+      end
+    end
+
+    def initialize(options={})
+      @options = options
+    end
+
+    def convert(input)
+      schema = Schema.new
+
+      command_parser = Groonga::Command::Parser.new
+      command_parser.on_command do |command|
+        case command.name
+        when "table_create"
+          schema.on_table_create_command(command)
+        when "column_create"
+          schema.on_column_create_command(command)
+        end
+      end
+
+      command_parser.on_load_value do |command, value|
+        command.original_source.clear
+      end
+
+      input.each_line do |line|
+        command_parser << line
+      end
+      command_parser.finish
+
+      schema.to_droonga_schema
+    end
+
+    class Schema
+      def initialize
+        @tables = {}
+      end
+
+      def on_table_create_command(command)
+        @tables[command[:name]] = Table.new(command)
+      end
+
+      def on_column_create_command(command)
+        table_name = command.table
+        table = @tables[table_name]
+        raise UnknownTable.new(table_name) if table.nil?
+        table.add_column(command[:name],
+                         Column.new(command))
+      end
+
+      def to_droonga_schema
+        droonga_schema = {}
+        @tables.each do |name, table|
+          droonga_schema[name] = table.to_droonga_schema
+        end
+        droonga_schema
+      end
+    end
+
+    class Table
+      def initialize(table_create_command)
+        @command = table_create_command
+        @columns = {}
+      end
+
+      def add_column(name, column)
+        @columns[name] = column
+      end
+
+      def to_droonga_schema
+        schema = {}
+        set_schema_item(schema, "type", type)
+        set_schema_item(schema, "keyType", key_type)
+        set_schema_item(schema, "tokenizer", tokenizer)
+        set_schema_item(schema, "normalizer", normalizer)
+        set_schema_item(schema, "columns", droonga_schema_columns)
+        schema
+      end
+
+      private
+      def set_schema_item(schema, key, value)
+        return if value.nil?
+        schema[key] = value
+      end
+
+      def type
+        if @command.table_no_key?
+          "Array"
+        elsif @command.table_hash_key?
+          "Hash"
+        elsif @command.table_pat_key?
+          "PatriciaTrie"
+        elsif @command.table_dat_key?
+          "DoubleArrayTrie"
+        else
+          "Hash"
+        end
+      end
+
+      def key_type
+        type = @command.key_type
+        case type
+        when /\AInt/, /\AUInt/
+          "Integer"
+        else
+          type
+        end
+      end
+
+      def tokenizer
+        @command.default_tokenizer
+      end
+
+      def normalizer
+        @command.normalizer
+      end
+
+      def droonga_schema_columns
+        return nil if @columns.empty?
+        schema = {}
+        @columns.each do |name, column|
+          schema[name] = column.to_droonga_schema
+        end
+        schema
+      end
+    end
+
+    class Column
+      def initialize(column_create_command)
+        @command = column_create_command
+      end
+
+      def to_droonga_schema
+        schema = {}
+        set_schema_item(schema, "type", type)
+        set_schema_item(schema, "valueType", value_type)
+        set_schema_item(schema, "vectorOptions", vector_options)
+        set_schema_item(schema, "indexOptions", index_options)
+        schema
+      end
+
+      private
+      def set_schema_item(schema, key, value)
+        return if value.nil?
+        schema[key] = value
+      end
+
+      def type
+        if @command.column_scalar?
+          "Scalar"
+        elsif @command.column_vector?
+          "Vector"
+        elsif @command.column_index?
+          "Index"
+        else
+          "Scalar"
+        end
+      end
+
+      def value_type
+        @command.type
+      end
+
+      def vector_options
+        return nil unless @command.column_vector?
+        {
+          "weight" => @command.with_weight?,
+        }
+      end
+
+      def index_options
+        return nil unless @command.column_index?
+        {
+          "section"  => @command.with_section?,
+          "weight"   => @command.with_weight?,
+          "position" => @command.with_position?,
+          "sources"  => @command.sources,
+        }
+      end
+    end
+  end
+end

data/lib/grn2drn/version.rb

@@ -14,5 +14,5 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 module Grn2Drn
-  VERSION = "1.0.0"
+  VERSION = "1.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grn2drn
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Droonga Project
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-09 00:00:00.000000000 Z
+date: 2014-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -113,6 +113,7 @@ email:
 - droonga@groonga.org
 executables:
 - grn2drn
+- grn2drn-schema
 extensions: []
 extra_rdoc_files: []
 files:
@@ -120,9 +121,12 @@ files:
 - Rakefile
 - Gemfile
 - grn2drn.gemspec
-- lib/grn2drn/converter.rb
+- lib/grn2drn/command-converter.rb
 - lib/grn2drn/version.rb
+- lib/grn2drn/error.rb
+- lib/grn2drn/schema-converter.rb
 - bin/grn2drn
+- bin/grn2drn-schema
 homepage: https://github.com/droonga/grn2drn
 licenses:
 - GPLv3 or later
@@ -143,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.0.13
 signing_key: 
 specification_version: 4
 summary: Grn2drn is a command to convert a *.grn dump file to a JSONs file for Droonga.