grn2drn 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e0a8d7c5929c03621ad56721c0cd286ce9f4496
+  data.tar.gz: 92308316da5be5c531c9620f7e6c19aec7b1481a
+SHA512:
+  metadata.gz: 205e98ee3714af39439967bf13ac5c6e554435cba3f6c03930525b5befeb38f1768f164ea3d6f488a4fa2c90bbce937ef2969c84d149c88800134c0b1c4b1f40
+  data.tar.gz: bed2c4438e9a1d00414f43bdc55c18ca97718bb7c697fba37c41131e1be50f97a1f76ae6099a7c3ff8a2d325715ca7720d1ebee5c3ab6b8f6cb405a80b17b5cd

data/Gemfile

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,38 @@
+# README
+
+## Name
+
+grn2drn
+
+## Description
+
+Grn2drn is a command to convert a *.grn dump file to a JSONs file for Droonga.
+
+## Install
+
+```
+% gem install grn2drn
+```
+
+## Basic usage
+
+```
+% grn2drn --dataset Droonga /path/to/grn/dump/file
+```
+
+## Mailing list
+
+* English: [groonga-talk@lists.sourceforge.net](https://lists.sourceforge.net/lists/listinfo/groonga-talk)
+* Japanese: [groonga-dev@lists.sourceforge.jp](http://lists.sourceforge.jp/mailman/listinfo/groonga-dev)
+
+## Thanks
+
+* ...
+
+## Copyright
+
+Copyright (c) 2014 Droonga Project
+
+## License
+
+GPLv3 or later. See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1,41 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2013  Droonga Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+task :default => :test
+
+require "bundler/gem_helper"
+require "packnga"
+
+base_dir = File.join(File.dirname(__FILE__))
+
+helper = Bundler::GemHelper.new(base_dir)
+def helper.version_tag
+  version
+end
+
+helper.install
+spec = helper.gemspec
+
+Packnga::DocumentTask.new(spec) do |task|
+  task.original_language = "en"
+  task.translate_languages = ["ja"]
+end
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb")
+end

data/bin/grn2drn

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "ostruct"
+require "optparse"
+
+require "grn2drn/converter"
+
+options = OpenStruct.new
+option_parser = OptionParser.new do |parser|
+  parser.separator("")
+  parser.separator("Converts Groonga commands to Droonga messages")
+
+  parser.separator("")
+  parser.separator("Required parameters:")
+  parser.on("--dataset=DATASET",
+            "Use DATASET as \"dataset\" field value") do |dataset|
+    options.dataset = dataset
+  end
+
+  parser.separator("")
+  parser.separator("Optional parameters:")
+  parser.on("--id-prefix=PREFIX",
+            "Use PREFIX as prefix of IDs") do |prefix|
+    options.id_prefix = prefix
+  end
+
+  parser.on("--date=DATE",
+            "Use DATE as \"date\" field value") do |date|
+    options.date = date
+  end
+
+  parser.on("--reply-to=TO",
+            "Use TO as \"replyTo\" field value",
+            "Responses from Droonga engine will be sent to TO.") do |to|
+    options.reply_to = to
+  end
+end
+args = option_parser.parse!(ARGV)
+
+if options.dataset.nil?
+  puts("--dataset option is missed.")
+  exit(false)
+end
+
+convert_options = {
+  :id_prefix => options.id_prefix,
+  :date => options.date,
+  :reply_to => options.reply_to,
+  :dataset => options.dataset,
+}
+converter = Grn2Drn::Converter.new(convert_options)
+
+source_file = args[0]
+result_file = args[1]
+
+input = nil
+if source_file.nil?
+  input = STDIN.read
+else
+  input = File.read(source_file)
+end
+
+result_file = args[1]
+
+if result_file.nil?
+  converter.convert(input) do |command|
+    puts(JSON.generate(command))
+  end
+else
+  File.open("w", result_file) do |file|
+    converter.convert(input) do |command|
+      file.puts(JSON.generate(command))
+    end
+  end
+end

data/grn2drn.gemspec

@@ -0,0 +1,52 @@
+# -*- mode: ruby; coding: utf-8 -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+clean_white_space = lambda do |entry|
+  entry.gsub(/(\A\n+|\n+\z)/, '') + "\n"
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "lib"))
+require "grn2drn/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "grn2drn"
+  spec.version = Grn2Drn::VERSION
+  spec.homepage = "https://github.com/droonga/grn2drn"
+  spec.authors = ["Droonga Project"]
+  spec.email = ["droonga@groonga.org"]
+  readme = File.read("README.md")
+  readme.force_encoding("UTF-8")
+  entries = readme.split(/^\#\#\s(.*)$/)
+  description = clean_white_space.call(entries[entries.index("Description") + 1])
+  spec.summary, spec.description, = description.split(/\n\n+/, 3)
+  spec.license = "GPLv3 or later"
+  spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
+  spec.files += Dir.glob("doc/text/**/*.rb")
+  spec.files += Dir.glob("lib/**/*.rb")
+  Dir.chdir("bin") do
+    spec.executables = Dir.glob("*")
+  end
+
+  spec.add_runtime_dependency("json")
+  spec.add_runtime_dependency("groonga-command-parser")
+
+  spec.add_development_dependency("bundler")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("test-unit")
+  spec.add_development_dependency("packnga")
+  spec.add_development_dependency("kramdown")
+end

data/lib/grn2drn/converter.rb

@@ -0,0 +1,160 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2013-2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "digest/sha1"
+require "time"
+require "json"
+
+require "groonga/command/parser"
+
+module Grn2Drn
+  class Converter
+    def initialize(options={})
+      @options = options
+      @count = 0
+
+      @command_parser = Groonga::Command::Parser.new
+    end
+
+    def convert(input, &block)
+      @command_parser.on_command do |command|
+        case command.name
+        when "table_create"
+          yield create_table_create_command(command)
+        when "table_remove"
+          yield create_table_remove_command(command)
+        when "column_create"
+          yield create_column_create_command(command)
+        when "select"
+          yield create_select_command(command)
+        end
+      end
+
+      parsed_values = nil
+      parsed_columns = nil
+      @command_parser.on_load_start do |command|
+        parsed_values = []
+        parsed_columns = nil
+      end
+      @command_parser.on_load_columns do |command, columns|
+        parsed_columns = columns
+      end
+      @command_parser.on_load_value do |command, value|
+        parsed_values << value
+      end
+      @command_parser.on_load_complete do |command|
+        command[:columns] ||= parsed_columns.join(",") if parsed_columns
+        command[:values] = parsed_values.to_json
+        split_load_command_to_add_commands(command, &block)
+      end
+
+      input.each_line do |line|
+        @command_parser << line
+      end
+      @command_parser.finish
+    end
+
+    private
+    def create_message(type, body)
+      id_prefix = @options[:id_prefix]
+      if id_prefix.nil?
+        id = new_unique_id
+      else
+        id = "#{id_prefix}:#{@count}"
+        @count += 1
+      end
+
+      {
+        "id" => id,
+        "date" => format_date(@options[:date] || Time.now),
+        "replyTo" => @options[:reply_to],
+        "dataset" => @options[:dataset],
+        "type" => type,
+        "body" => body,
+      }
+    end
+
+    def new_unique_id
+      now = Time.now
+      now_msec = now.to_i * 1000 + now.usec
+      random_string = rand(36 ** 16).to_s(36) # Base36
+      Digest::SHA1.hexdigest("#{now_msec}:#{random_string}")
+    end
+
+    def format_date(time)
+      time.iso8601
+    end
+
+    def stringify_keys(hash)
+      stringified_hash = {}
+      hash.each do |key, value|
+        stringified_hash[key.to_s] = value
+      end
+      stringified_hash
+    end
+
+    def command_to_body(command)
+      stringify_keys(command.arguments)
+    end
+
+    def create_table_create_command(command)
+      create_message("table_create", command_to_body(command))
+    end
+
+    def create_table_remove_command(command)
+      create_message("table_remove", command_to_body(command))
+    end
+
+    def create_column_create_command(command)
+      create_message("column_create", command_to_body(command))
+    end
+
+    def split_load_command_to_add_commands(command, &block)
+      columns = command.columns
+      values = command[:values]
+      values = JSON.parse(values)
+      values.each do |record|
+        body = {
+          "table" => command[:table],
+        }
+
+        if record.is_a?(Hash)
+          record = record.dup
+          body["key"] = record.delete("_key")
+          record_values = record
+        else
+          record_values = {}
+          record.each_with_index do |value, column_index|
+            column = columns[column_index]
+            if column == "_key"
+              body["key"] = value
+            else
+              record_values[column] = value
+            end
+          end
+        end
+        body["values"] = record_values unless record_values.empty?
+
+        yield create_message("add", body)
+      end
+    end
+
+    def create_select_command(command)
+      create_message("select", command_to_body(command))
+    end
+  end
+end

data/lib/grn2drn/version.rb

@@ -0,0 +1,18 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+module Grn2Drn
+  VERSION = "1.0.0"
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: grn2drn
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Droonga Project
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: groonga-command-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: packnga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kramdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- droonga@groonga.org
+executables:
+- grn2drn
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- Gemfile
+- grn2drn.gemspec
+- lib/grn2drn/converter.rb
+- lib/grn2drn/version.rb
+- bin/grn2drn
+homepage: https://github.com/droonga/grn2drn
+licenses:
+- GPLv3 or later
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Grn2drn is a command to convert a *.grn dump file to a JSONs file for Droonga.
+test_files: []
+has_rdoc: