rroonga 0.9.2-mswin32

data/benchmark/common.rb

@@ -0,0 +1,49 @@
+require 'benchmark'
+require 'tempfile'
+
+def memory_usage()
+  status = `cat /proc/#{$$}/status`
+  lines = status.split("\n")
+  lines.each do |line|
+    if line =~ /^VmRSS:/
+      line.gsub!(/.*:\s*(\d+).*/, '\1')
+      return line.to_i / 1024.0
+    end
+  end
+  return -1;
+end
+
+@items = []
+
+def item(label, &block)
+  @items << [label, block]
+end
+
+def report(index=0)
+  width = @items.collect do |label, _|
+    label.length
+  end.max
+
+  label, block = @items[index]
+  if label.nil?
+    puts "unavailable report ID: #{index}"
+    puts "available IDs:"
+    @items.each_with_index do |(label, block), i|
+      puts "#{i}: #{label}"
+    end
+    exit 1
+  end
+
+  if index.zero?
+    puts(" " * (width - 1) + Benchmark::Tms::CAPTION.rstrip + "memory".rjust(14))
+  end
+  # GC.disable
+  before = memory_usage
+  result = Benchmark.measure(&block)
+  # GC.enable
+  GC.start
+  size = memory_usage - before
+
+  formatted_size = "%10.3f" % size
+  puts "#{label.ljust(width)} #{result.to_s.strip} #{formatted_size}MB"
+end

data/benchmark/read-write-many-small-items.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/ruby
+
+# This benchmark is based on Tokyo Cabinet's benchmark at
+# http://alpha.mixi.co.jp/blog/?p=791
+#
+# On my environment at 2009/07/14:
+# % for x in {0..9}; do ruby benchmark/read-write-many-small-items.rb $x; done
+#                           user     system      total        real        memory
+# Hash                  0.850000   0.170000   1.020000 (  1.050703)     46.957MB
+# groonga: Hash: memory 1.080000   0.180000   1.260000 (  1.251981)     30.152MB
+# groonga: Trie: memory 1.300000   0.120000   1.420000 (  1.429549)     22.199MB
+# groonga: Hash: file   1.070000   0.190000   1.260000 (  1.276910)     30.156MB
+# groonga: Trie: file   1.280000   0.150000   1.430000 (  1.423066)     22.203MB
+# Localmemcache: file   1.320000   0.150000   1.470000 (  1.497013)     45.984MB
+# TC: Hash: memory      0.940000   0.250000   1.190000 (  1.248141)     48.758MB
+# TC: Tree: memory      0.890000   0.140000   1.030000 (  1.067693)     37.270MB
+# TC: Hash: file        1.740000   2.390000   4.130000 (  4.141555)      8.848MB
+# TC: Tree: file        1.290000   0.180000   1.470000 (  1.476853)     12.805MB
+#
+# CPU: Intel(R) Core(TM)2 Duo 2.33GHz
+# Memory: 2GB
+# Ruby: Debian GNU/Linux sid at 2009/07/14:
+#       ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# groonga: HEAD at 2009/07/14: fdaf58df5dd0195c10624eabee3e3f522f4af3f9
+# Ruby/Groonga: trunk at 2009/07/14: r479
+# Localmemcache: HEAD at 2009/07/14: 3121629016344dfd10f7533ca8ec68a0006cca21
+# Tokyo Cabinet: 1.4.29
+# Tokyo Cabinet Ruby: 1.27
+#
+# NOTE:
+# * groonga, Localmemcache and Tokyo Cabinet are built with "-O3" option.
+# * Ruby bindings of them are built with "-O2" option.
+
+require File.join(File.dirname(__FILE__), "common.rb")
+
+n = 500000
+
+values = []
+n.times do |i|
+  values << "%08d" % i
+end
+
+item("Hash") do
+  @hash = {}
+  values.each do |value|
+    @hash[value] = value
+    @hash[value]
+  end
+end
+
+begin
+  base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+  $LOAD_PATH.unshift(File.join(base_dir, "ext"))
+  $LOAD_PATH.unshift(File.join(base_dir, "lib"))
+
+  require 'groonga'
+  tmp_dir = "/tmp/groonga"
+  FileUtils.rm_rf(tmp_dir)
+  FileUtils.mkdir(tmp_dir)
+  @database = Groonga::Database.create(:path => "#{tmp_dir}/db")
+
+  item("groonga: Hash: file") do
+    @hash = Groonga::Hash.create(:name => "Hash",
+                                 :key_type => "ShortText")
+    column_name = "value"
+    @column = @hash.define_column(column_name, "ShortText")
+    values.each do |value|
+      @hash.set_column_value(value, column_name, value)
+      @hash.column_value(value, column_name)
+    end
+  end
+
+  item("groonga: Trie: file") do
+    @trie = Groonga::PatriciaTrie.create(:name => "PatriciaTrie",
+                                         :key_type => "ShortText")
+    column_name = "value"
+    @column = @trie.define_column(column_name, "ShortText")
+    values.each do |value|
+      @trie.set_column_value(value, column_name, value)
+      @trie.column_value(value, column_name)
+    end
+  end
+rescue LoadError
+end
+
+begin
+  require 'localmemcache'
+
+  item("Localmemcache: file") do
+    LocalMemCache.drop(:namespace => "read-write-many-small-items",
+                       :force => true)
+    @db = LocalMemCache.new(:namespace => "read-write-many-small-items")
+    values.each do |value|
+      @db[value] = value
+      @db[value]
+    end
+  end
+rescue LoadError
+end
+
+begin
+  require 'tokyocabinet'
+
+  item("TC: Hash: memory") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("*#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+      @db.get(value)
+    end
+  end
+
+  item("TC: Tree: memory") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("+#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+      @db.get(value)
+    end
+  end
+
+  hash_file = Tempfile.new(["tc-hash", ".tch"])
+  item("TC: Hash: file") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("#{hash_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+      @db.get(value)
+    end
+  end
+
+  tree_file = Tempfile.new(["tc-tree", ".tcb"])
+  item("TC: Tree: file") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("#{tree_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+      @db.get(value)
+    end
+  end
+rescue LoadError
+end
+
+report(Integer(ARGV[0] || 0))

data/benchmark/write-many-small-items.rb

@@ -0,0 +1,135 @@
+#!/usr/bin/ruby
+
+# This benchmark is based on Tokyo Cabinet's benchmark at
+# http://alpha.mixi.co.jp/blog/?p=791
+#
+# On my environment at 2009/07/14:
+# % for x in {0..9}; do ruby benchmark/write-many-small-items.rb $x; done
+#                           user     system      total        real        memory
+# Hash                  0.650000   0.130000   0.780000 (  0.799843)     46.957MB
+# groonga: Hash: memory 0.650000   0.130000   0.780000 (  0.781058)     23.477MB
+# groonga: Trie: memory 0.690000   0.180000   0.870000 (  0.862132)     15.516MB
+# groonga: Hash: file   0.660000   0.120000   0.780000 (  0.780952)     23.480MB
+# groonga: Trie: file   0.660000   0.190000   0.850000 (  0.867515)     15.520MB
+# Localmemcache: file   0.900000   0.150000   1.050000 (  1.052692)     39.312MB
+# TC: Hash: memory      0.480000   0.150000   0.630000 (  0.636297)     42.062MB
+# TC: Tree: memory      0.440000   0.150000   0.590000 (  0.593117)     30.570MB
+# TC: Hash: file        1.000000   1.820000   2.820000 (  2.989515)      2.160MB
+# TC: Tree: file        0.720000   0.130000   0.850000 (  0.877557)      6.102MB
+#
+# CPU: Intel(R) Core(TM)2 Duo 2.33GHz
+# Memory: 2GB
+# Ruby: Debian GNU/Linux sid at 2009/07/14:
+#       ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
+# groonga: HEAD at 2009/07/14: fdaf58df5dd0195c10624eabee3e3f522f4af3f9
+# Ruby/Groonga: trunk at 2009/07/14: r479
+# Localmemcache: HEAD at 2009/07/14: 3121629016344dfd10f7533ca8ec68a0006cca21
+# Tokyo Cabinet: 1.4.29
+# Tokyo Cabinet Ruby: 1.27
+#
+# NOTE:
+# * groonga, Localmemcache and Tokyo Cabinet are built with "-O3" option.
+# * Ruby bindings of them are built with "-O2" option.
+
+require File.join(File.dirname(__FILE__), "common.rb")
+
+n = 500000
+
+values = []
+n.times do |i|
+  values << "%08d" % i
+end
+
+item("Hash") do
+  @hash = {}
+  values.each do |value|
+    @hash[value] = value
+  end
+end
+
+begin
+  base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+  $LOAD_PATH.unshift(File.join(base_dir, "ext"))
+  $LOAD_PATH.unshift(File.join(base_dir, "lib"))
+
+  require 'groonga'
+  tmp_dir = "/tmp/groonga"
+  FileUtils.rm_rf(tmp_dir)
+  FileUtils.mkdir(tmp_dir)
+  @database = Groonga::Database.create(:path => "#{tmp_dir}/db")
+
+  item("groonga: Hash: file") do
+    @hash = Groonga::Hash.create(:name => "Hash",
+                                 :key_type => "ShortText")
+    column_name = "value"
+    @column = @hash.define_column(column_name, "ShortText")
+    values.each do |value|
+      @hash.set_column_value(value, column_name, value)
+    end
+  end
+
+  item("groonga: Trie: file") do
+    @trie = Groonga::PatriciaTrie.create(:name => "PatriciaTrie",
+                                         :key_type => "ShortText")
+    column_name = "value"
+    @column = @trie.define_column(column_name, "ShortText")
+    values.each do |value|
+      @trie.set_column_value(value, column_name, value)
+    end
+  end
+rescue LoadError
+end
+
+begin
+  require 'localmemcache'
+
+  item("Localmemcache: file") do
+    LocalMemCache.drop(:namespace => "write-many-small-items", :force => true)
+    @db = LocalMemCache.new(:namespace => "write-many-small-items")
+    values.each do |value|
+      @db[value] = value
+    end
+  end
+rescue LoadError
+end
+
+begin
+  require 'tokyocabinet'
+
+  item("TC: Hash: memory") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("*#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+    end
+  end
+
+  item("TC: Tree: memory") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("+#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+    end
+  end
+
+  hash_file = Tempfile.new(["tc-hash", ".tch"])
+  item("TC: Hash: file") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("#{hash_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+    end
+  end
+
+  tree_file = Tempfile.new(["tc-tree", ".tcb"])
+  item("TC: Tree: file") do
+    @db = TokyoCabinet::ADB::new
+    @db.open("#{tree_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
+    values.each do |value|
+      @db.put(value, value)
+    end
+  end
+rescue LoadError
+end
+
+report(Integer(ARGV[0] || 0))

data/example/bookmark.rb

@@ -0,0 +1,161 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+base_dir = File.join(File.dirname(__FILE__), "..")
+groonga_ext_dir = File.join(base_dir, 'ext')
+groonga_lib_dir = File.join(base_dir, 'lib')
+$LOAD_PATH.unshift(groonga_ext_dir)
+$LOAD_PATH.unshift(groonga_lib_dir)
+
+begin
+  require "groonga"
+rescue LoadError
+  require "rubygems"
+  require "groonga"
+end
+
+require 'time'
+
+# Groonga::Logger.register(:level => :debug) do |level, time, title, message, location|
+#   p [level, time, title, message, location]
+# end
+
+$KCODE = "UTF-8"
+Groonga::Context.default_options = {:encoding => :utf8}
+
+path = ARGV[0]
+if path.nil?
+  require 'tmpdir'
+  require 'fileutils'
+  temporary_directory = File.join(Dir.tmpdir, "ruby-groonga")
+  FileUtils.mkdir_p(temporary_directory)
+  at_exit {FileUtils.rm_rf(temporary_directory)}
+  path = File.join(temporary_directory, "db")
+end
+persistent = true
+
+p Groonga::Database.create(:path => path)
+
+p(items = Groonga::Hash.create(:name => "Items",
+                               :key_type => "ShortText",
+                               :persistent => persistent))
+
+p items.add("http://ja.wikipedia.org/wiki/Ruby")
+p items.add("http://www.ruby-lang.org/")
+
+p items.define_column("title", "Text", :persistent => persistent)
+
+p(terms = Groonga::Hash.create(:name => "Terms",
+                               :key_type => "ShortText",
+                               :persistent => persistent,
+                               :default_tokenizer => "TokenBigram"))
+p terms.define_index_column("item_title", items,
+                            :persistent => persistent,
+                            :with_weight => true,
+                            :with_section => true,
+                            :with_position => true,
+                            :source => "Items.title")
+
+p items.find("http://ja.wikipedia.org/wiki/Ruby")["title"] = "Ruby"
+p items.find("http://www.ruby-lang.org/")["title"] = "オブジェクト指向スクリプト言語Ruby"
+
+p(users = Groonga::Hash.create(:name => "Users",
+                               :key_type => "ShortText",
+                               :persistent => persistent))
+p users.define_column("name", "Text",
+                      :persistent => persistent)
+
+p(comments = Groonga::Array.create(:name => "Comments",
+                                   :persistent => persistent))
+p comments.define_column("item", items)
+p comments.define_column("author", users)
+p comments.define_column("content", "Text")
+p comments.define_column("issued", "Time")
+
+p terms.define_index_column("comment_content", comments,
+                            :persistent => persistent,
+                            :with_weight => true,
+                            :with_section => true,
+                            :with_position => true,
+                            :source => "Comments.content")
+
+p users.add("moritan", :name => "モリタン")
+p users.add("taporobo", :name => "タポロボ")
+
+
+p items.find("http://d.hatena.ne.jp/brazil/20050829/1125321936")
+
+p items.add("http://d.hatena.ne.jp/brazil/20050829/1125321936",
+            :title => "[翻訳]JavaScript: 世界で最も誤解されたプログラミング言語")
+
+p comments.add(:item => "http://d.hatena.ne.jp/brazil/20050829/1125321936",
+               :author => "moritan",
+               :content => "JavaScript LISP",
+               :issued => 1187430026)
+
+@items = items
+@comments = comments
+def add_bookmark(url, title, author, content, issued)
+  item = @items.find(url) || @items.add(url, :title => title)
+  @comments.add(:item => item,
+                :author => author,
+                :content => content,
+                :issued => issued)
+end
+
+p add_bookmark("http://practical-scheme.net/docs/cont-j.html",
+               "なんでも継続", "moritan", "継続 LISP Scheme", 1187568692)
+p add_bookmark("http://d.hatena.ne.jp/higepon/20070815/1187192864",
+               "末尾再帰", "taporobo", "末尾再帰 Scheme LISP", 1187568793)
+p add_bookmark("http://practical-scheme.net/docs/cont-j.html",
+               "なんでも継続", "taporobo", "トランポリン LISP continuation",
+               1187568692.98765)
+p add_bookmark("http://jp.rubyist.net/managinze",
+               "るびま", "moritan", "Ruby ドキュメント",
+               Time.now)
+p add_bookmark("http://jp.rubyist.net/managinze",
+               "るびま", "taporobo", "Ruby 雑誌",
+               Time.now)
+p add_bookmark("http://groonga.rubyforge.org/",
+               "ラングバ", "moritan", "Ruby groonga",
+               Time.parse("2009-07-19"))
+
+
+records = comments.select do |record|
+  record["content"] =~ "LISP"
+end
+
+records.each do |record|
+  record = record.key
+  p [record.id,
+     record[".issued"],
+     record[".item.title"],
+     record[".author.name"],
+     record[".content"]]
+end
+
+p records
+
+records.sort([{:key => ".issued", :order => "descending"}]).each do |record|
+  record = record.key
+  p [record.id,
+     record[".issued"],
+     record[".item.title"],
+     record[".author.name"],
+     record[".content"]]
+end
+
+records.group([".item"]).each do |record|
+  item = record.key
+  p [record.n_sub_records,
+     item.key,
+     item[".title"]]
+end
+
+p ruby_comments = @comments.select {|record| record["content"] =~ "Ruby"}
+p ruby_items = @items.select("*W1:50 title:@Ruby")
+
+p ruby_items = ruby_comments.group([".item"]).union!(ruby_items)
+ruby_items.sort([{:key => "._score", :order => "descending"}]).each do |record|
+  p [record["._score"], record[".title"]]
+end