groonga 0.9.1 → 1.0.0

data/benchmark/write-many-small-items.rb

@@ -1,135 +0,0 @@
-#!/usr/bin/ruby
-
-# This benchmark is based on Tokyo Cabinet's benchmark at
-# http://alpha.mixi.co.jp/blog/?p=791
-#
-# On my environment at 2009/07/14:
-# % for x in {0..9}; do ruby benchmark/write-many-small-items.rb $x; done
-#                           user     system      total        real        memory
-# Hash                  0.650000   0.130000   0.780000 (  0.799843)     46.957MB
-# groonga: Hash: memory 0.650000   0.130000   0.780000 (  0.781058)     23.477MB
-# groonga: Trie: memory 0.690000   0.180000   0.870000 (  0.862132)     15.516MB
-# groonga: Hash: file   0.660000   0.120000   0.780000 (  0.780952)     23.480MB
-# groonga: Trie: file   0.660000   0.190000   0.850000 (  0.867515)     15.520MB
-# Localmemcache: file   0.900000   0.150000   1.050000 (  1.052692)     39.312MB
-# TC: Hash: memory      0.480000   0.150000   0.630000 (  0.636297)     42.062MB
-# TC: Tree: memory      0.440000   0.150000   0.590000 (  0.593117)     30.570MB
-# TC: Hash: file        1.000000   1.820000   2.820000 (  2.989515)      2.160MB
-# TC: Tree: file        0.720000   0.130000   0.850000 (  0.877557)      6.102MB
-#
-# CPU: Intel(R) Core(TM)2 Duo 2.33GHz
-# Memory: 2GB
-# Ruby: Debian GNU/Linux sid at 2009/07/14:
-#       ruby 1.8.7 (2009-06-12 patchlevel 174) [x86_64-linux]
-# groonga: HEAD at 2009/07/14: fdaf58df5dd0195c10624eabee3e3f522f4af3f9
-# Ruby/Groonga: trunk at 2009/07/14: r479
-# Localmemcache: HEAD at 2009/07/14: 3121629016344dfd10f7533ca8ec68a0006cca21
-# Tokyo Cabinet: 1.4.29
-# Tokyo Cabinet Ruby: 1.27
-#
-# NOTE:
-# * groonga, Localmemcache and Tokyo Cabinet are built with "-O3" option.
-# * Ruby bindings of them are built with "-O2" option.
-
-require File.join(File.dirname(__FILE__), "common.rb")
-
-n = 500000
-
-values = []
-n.times do |i|
-  values << "%08d" % i
-end
-
-item("Hash") do
-  @hash = {}
-  values.each do |value|
-    @hash[value] = value
-  end
-end
-
-begin
-  base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-  $LOAD_PATH.unshift(File.join(base_dir, "ext"))
-  $LOAD_PATH.unshift(File.join(base_dir, "lib"))
-
-  require 'groonga'
-  tmp_dir = "/tmp/groonga"
-  FileUtils.rm_rf(tmp_dir)
-  FileUtils.mkdir(tmp_dir)
-  @database = Groonga::Database.create(:path => "#{tmp_dir}/db")
-
-  item("groonga: Hash: file") do
-    @hash = Groonga::Hash.create(:name => "Hash",
-                                 :key_type => "ShortText")
-    column_name = "value"
-    @column = @hash.define_column(column_name, "ShortText")
-    values.each do |value|
-      @hash.set_column_value(value, column_name, value)
-    end
-  end
-
-  item("groonga: Trie: file") do
-    @trie = Groonga::PatriciaTrie.create(:name => "PatriciaTrie",
-                                         :key_type => "ShortText")
-    column_name = "value"
-    @column = @trie.define_column(column_name, "ShortText")
-    values.each do |value|
-      @trie.set_column_value(value, column_name, value)
-    end
-  end
-rescue LoadError
-end
-
-begin
-  require 'localmemcache'
-
-  item("Localmemcache: file") do
-    LocalMemCache.drop(:namespace => "write-many-small-items", :force => true)
-    @db = LocalMemCache.new(:namespace => "write-many-small-items")
-    values.each do |value|
-      @db[value] = value
-    end
-  end
-rescue LoadError
-end
-
-begin
-  require 'tokyocabinet'
-
-  item("TC: Hash: memory") do
-    @db = TokyoCabinet::ADB::new
-    @db.open("*#bnum=#{n}#mode=wct#xmsiz=0")
-    values.each do |value|
-      @db.put(value, value)
-    end
-  end
-
-  item("TC: Tree: memory") do
-    @db = TokyoCabinet::ADB::new
-    @db.open("+#bnum=#{n}#mode=wct#xmsiz=0")
-    values.each do |value|
-      @db.put(value, value)
-    end
-  end
-
-  hash_file = Tempfile.new(["tc-hash", ".tch"])
-  item("TC: Hash: file") do
-    @db = TokyoCabinet::ADB::new
-    @db.open("#{hash_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
-    values.each do |value|
-      @db.put(value, value)
-    end
-  end
-
-  tree_file = Tempfile.new(["tc-tree", ".tcb"])
-  item("TC: Tree: file") do
-    @db = TokyoCabinet::ADB::new
-    @db.open("#{tree_file.path}#bnum=#{n}#mode=wct#xmsiz=0")
-    values.each do |value|
-      @db.put(value, value)
-    end
-  end
-rescue LoadError
-end
-
-report(Integer(ARGV[0] || 0))

data/example/bookmark.rb

@@ -1,161 +0,0 @@
-#!/usr/bin/env ruby
-# -*- coding: utf-8 -*-
-
-base_dir = File.join(File.dirname(__FILE__), "..")
-groonga_ext_dir = File.join(base_dir, 'ext')
-groonga_lib_dir = File.join(base_dir, 'lib')
-$LOAD_PATH.unshift(groonga_ext_dir)
-$LOAD_PATH.unshift(groonga_lib_dir)
-
-begin
-  require "groonga"
-rescue LoadError
-  require "rubygems"
-  require "groonga"
-end
-
-require 'time'
-
-# Groonga::Logger.register(:level => :debug) do |level, time, title, message, location|
-#   p [level, time, title, message, location]
-# end
-
-$KCODE = "UTF-8"
-Groonga::Context.default_options = {:encoding => :utf8}
-
-path = ARGV[0]
-if path.nil?
-  require 'tmpdir'
-  require 'fileutils'
-  temporary_directory = File.join(Dir.tmpdir, "ruby-groonga")
-  FileUtils.mkdir_p(temporary_directory)
-  at_exit {FileUtils.rm_rf(temporary_directory)}
-  path = File.join(temporary_directory, "db")
-end
-persistent = true
-
-p Groonga::Database.create(:path => path)
-
-p(items = Groonga::Hash.create(:name => "Items",
-                               :key_type => "ShortText",
-                               :persistent => persistent))
-
-p items.add("http://ja.wikipedia.org/wiki/Ruby")
-p items.add("http://www.ruby-lang.org/")
-
-p items.define_column("title", "Text", :persistent => persistent)
-
-p(terms = Groonga::Hash.create(:name => "Terms",
-                               :key_type => "ShortText",
-                               :persistent => persistent,
-                               :default_tokenizer => "TokenBigram"))
-p terms.define_index_column("item_title", items,
-                            :persistent => persistent,
-                            :with_weight => true,
-                            :with_section => true,
-                            :with_position => true,
-                            :source => "Items.title")
-
-p items.find("http://ja.wikipedia.org/wiki/Ruby")["title"] = "Ruby"
-p items.find("http://www.ruby-lang.org/")["title"] = "オブジェクト指向スクリプト言語Ruby"
-
-p(users = Groonga::Hash.create(:name => "Users",
-                               :key_type => "ShortText",
-                               :persistent => persistent))
-p users.define_column("name", "Text",
-                      :persistent => persistent)
-
-p(comments = Groonga::Array.create(:name => "Comments",
-                                   :persistent => persistent))
-p comments.define_column("item", items)
-p comments.define_column("author", users)
-p comments.define_column("content", "Text")
-p comments.define_column("issued", "Time")
-
-p terms.define_index_column("comment_content", comments,
-                            :persistent => persistent,
-                            :with_weight => true,
-                            :with_section => true,
-                            :with_position => true,
-                            :source => "Comments.content")
-
-p users.add("moritan", :name => "モリタン")
-p users.add("taporobo", :name => "タポロボ")
-
-
-p items.find("http://d.hatena.ne.jp/brazil/20050829/1125321936")
-
-p items.add("http://d.hatena.ne.jp/brazil/20050829/1125321936",
-            :title => "[翻訳]JavaScript: 世界で最も誤解されたプログラミング言語")
-
-p comments.add(:item => "http://d.hatena.ne.jp/brazil/20050829/1125321936",
-               :author => "moritan",
-               :content => "JavaScript LISP",
-               :issued => 1187430026)
-
-@items = items
-@comments = comments
-def add_bookmark(url, title, author, content, issued)
-  item = @items.find(url) || @items.add(url, :title => title)
-  @comments.add(:item => item,
-                :author => author,
-                :content => content,
-                :issued => issued)
-end
-
-p add_bookmark("http://practical-scheme.net/docs/cont-j.html",
-               "なんでも継続", "moritan", "継続 LISP Scheme", 1187568692)
-p add_bookmark("http://d.hatena.ne.jp/higepon/20070815/1187192864",
-               "末尾再帰", "taporobo", "末尾再帰 Scheme LISP", 1187568793)
-p add_bookmark("http://practical-scheme.net/docs/cont-j.html",
-               "なんでも継続", "taporobo", "トランポリン LISP continuation",
-               1187568692.98765)
-p add_bookmark("http://jp.rubyist.net/managinze",
-               "るびま", "moritan", "Ruby ドキュメント",
-               Time.now)
-p add_bookmark("http://jp.rubyist.net/managinze",
-               "るびま", "taporobo", "Ruby 雑誌",
-               Time.now)
-p add_bookmark("http://groonga.rubyforge.org/",
-               "ラングバ", "moritan", "Ruby groonga",
-               Time.parse("2009-07-19"))
-
-
-records = comments.select do |record|
-  record["content"] =~ "LISP"
-end
-
-records.each do |record|
-  record = record.key
-  p [record.id,
-     record[".issued"],
-     record[".item.title"],
-     record[".author.name"],
-     record[".content"]]
-end
-
-p records
-
-records.sort([{:key => ".issued", :order => "descending"}]).each do |record|
-  record = record.key
-  p [record.id,
-     record[".issued"],
-     record[".item.title"],
-     record[".author.name"],
-     record[".content"]]
-end
-
-records.group([".item"]).each do |record|
-  item = record.key
-  p [record.n_sub_records,
-     item.key,
-     item[".title"]]
-end
-
-p ruby_comments = @comments.select {|record| record["content"] =~ "Ruby"}
-p ruby_items = @items.select("*W1:50 title:@Ruby")
-
-p ruby_items = ruby_comments.group([".item"]).union!(ruby_items)
-ruby_items.sort([{:key => "._score", :order => "descending"}]).each do |record|
-  p [record["._score"], record[".title"]]
-end

data/example/index-html.rb

@@ -1,89 +0,0 @@
-#!/usr/bin/env ruby
-
-if ARGV.size < 2 or ARGV.find {|option| option == "-h" or option == "--help"}
-  puts "Usage: #{$0} DATABASE_FILE [FILE_OR_DIRECTORY ...]"
-  exit
-end
-
-require 'pathname'
-
-base_directory = Pathname(__FILE__).dirname + ".."
-$LOAD_PATH.unshift((base_directory + "ext").to_s)
-$LOAD_PATH.unshift((base_directory + "lib").to_s)
-
-require 'rubygems'
-require 'groonga'
-require 'nokogiri'
-
-database_file, *targets = ARGV
-
-database_file = Pathname(database_file)
-database_directory = database_file.dirname
-database_directory.mkpath unless database_directory.exist?
-
-if database_file.exist?
-  Groonga::Database.open(database_file.to_s)
-else
-  Groonga::Database.create(:path => database_file.to_s)
-  Groonga::Schema.define do |schema|
-    schema.create_table("documents") do |table|
-      table.string("title")
-      table.text("content")
-      table.string("path")
-      table.time("last-modified")
-    end
-
-    schema.create_table("terms",
-                        :type => :patricia_trie,
-                        :key_normalize => true,
-                        :default_tokenizer => "TokenBigram") do |table|
-      table.index("documents.title")
-      table.index("documents.content")
-    end
-  end
-end
-
-documents = Groonga::Context.default["documents"]
-
-targets.each do |target|
-  target = Pathname(target)
-  target.find do |path|
-    throw :prune if path.basename.to_s == ".svn"
-    if path.file? and path.extname == ".html"
-      path.open do |html|
-        values = {:path => path.relative_path_from(target).to_s}
-        _documents = documents.select do |record|
-          record["path"] == values[:path]
-        end
-        if _documents.size.zero?
-          document = documents.add
-        else
-          document = _documents.to_a[0].key
-        end
-
-        html_document = Nokogiri::HTML(html)
-        html_document.css("title").each do |title|
-          values[:title] = title.text
-        end
-        contents = []
-        html_document.css("body").each do |body|
-          contents << body.text
-        end
-        html_document.css("img").each do |image|
-          image_content = []
-          title = image['title']
-          alt = image['alt']
-          image_content << title if title and !title.empty?
-          image_content << alt if alt and !alt.empty?
-          contents.concat(image_content) unless image_content.empty?
-        end
-        values[:content] = contents.join("\n")
-        values["last-modified"] = path.mtime
-
-        values.each do |key, value|
-          document[key] = value
-        end
-      end
-    end
-  end
-end

data/example/search/config.ru

@@ -1,211 +0,0 @@
-# -*- mode: ruby; coding: utf-8 -*-
-
-base_dir = File.join(File.dirname(__FILE__), "..", "..")
-$LOAD_PATH.unshift(File.join(base_dir, "ext"))
-$LOAD_PATH.unshift(File.join(base_dir, "lib"))
-
-require 'rubygems'
-require 'rack'
-require 'groonga'
-
-use Rack::CommonLogger
-use Rack::Static, :urls => ["/css", "/images"], :root => "public"
-
-Groonga::Database.new("data/database")
-
-class Searcher
-  include Rack::Utils
-
-  def initialize
-    @documents = Groonga::Context.default["documents"]
-  end
-
-  def call(env)
-    request = Rack::Request.new(env)
-    response = Rack::Response.new
-    response["Content-Type"] = "text/html; charset=UTF-8"
-    if /\/\z/ !~ request.path_info
-      request.path_info += "/"
-      response.redirect(request.url)
-      return response.to_a
-    end
-
-    response.write(<<-EOH)
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-       "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="ja" lang="ja">
-<head>
-  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
-  <meta name="robot" content="noindex,nofollow" />
-  <title>groongaで全文検索</title>
-  <link rel="stylesheet" href="css/groonga.css" type="text/css" media="all" />
-</head>
-<body>
-<div class="header">
-  <h1>groongaで全文検索</h1>
-</div>
-
-<div class="content">
-EOH
-
-    render_search_box(request, response)
-    render_search_result(request, response)
-
-    response.write(<<-EOF)
-</div>
-
-<div class="footer">
-<p class="powered-by-groonga">
-  <a href="http://groonga.org/">groonga</a>
-  #{Groonga::VERSION.join('.')}を使っています。
-</p>
-<p class="powered-by-ruby-groonga">
-  <a href="http://groonga.rubyforge.org/">Ruby/groonga</a>
-  #{Groonga::BINDINGS_VERSION.join('.')}を使っています。
-</p>
-</div>
-
-</body>
-</html>
-EOF
-    response.to_a
-  end
-
-  private
-  def query(request)
-    request['query'] || ''
-  end
-
-  def page(request)
-    (request['page'] || 0).to_i
-  end
-
-  def render_search_box(request, response)
-    response.write(<<-EOF)
-<form method="get" action="./">
-  <p>
-    <a href="."><img src="images/mini-groonga.png" alt="groonga" /></a>
-    <input name="query" type="text" value="#{escape_html(query(request))}" />
-    <input type="submit" value="検索" />
-  </p>
-</form>
-EOF
-  end
-
-  def render_search_result(request, response)
-    _query = query(request)
-    _page = page(request)
-    limit = 20
-    if _query.empty?
-      records = []
-      response.write(<<-EOS)
-  <div class='search-summary'>
-    <p>Rubyでgroonga使って全文検索</p>
-  </div>
-EOS
-    else
-      options = {}
-      before = Time.now
-      records = @documents.select do |record|
-        record["content"].match(_query)
-      end
-      total_records = records.size
-      records = records.sort([[".:score", "descending"],
-                              [".last-modified", "descending"]],
-                             :offset => _page * limit,
-                             :limit => limit)
-      elapsed = Time.now - before
-
-      response.write(<<-EOS)
-  <div class='search-summary'>
-    <p>
-      <span class="keyword">#{escape_html(query(request))}</span>の検索結果:
-      <span class="total-entries">#{total_records}</span>件中
-      <span class="display-range">
-        #{total_records.zero? ? 0 : (_page * limit) + 1}
-        -
-        #{(_page * limit) + records.size}
-      </span>
-      件（#{elapsed}秒）
-    </p>
-  </div>
-EOS
-    end
-
-    response.write("  <div class='records'>\n")
-    records.each do |record|
-      render_record(request, response, record)
-    end
-    response.write("  </div>\n")
-
-    render_pagination(request, response, _page, limit, total_records)
-  end
-
-  def render_record(request, response, record)
-    response.write("    <div class='record'>\n")
-    href = escape_html(record['.path'])
-    title = escape_html(record['.title'])
-    last_modified = escape_html(record['.last-modified'].iso8601)
-    score = record.score
-    response.write("      <h2><a href='#{href}'>#{title}</a>(#{score})</h2>\n")
-    render_snippet(request, response, record)
-    response.write(<<-EOM)
-      <p class="metadata">
-        <span class="url">#{unescape(href)}</span>
-        -
-        <span class="last-modified">#{last_modified}</span>
-      </p>
-EOM
-    response.write("    </div>\n")
-  end
-
-  def render_snippet(request, response, record)
-    expression = record.table.expression
-    snippet = expression.snippet([["<span class=\"keyword\">", "</span>"]],
-                                 :width => 100,
-                                 :html_escape => true,
-                                 :normalize => true)
-    separator = "\n<span class='separator'>...</span>\n"
-    response.write(<<-EOS)
-      <p class="snippet">
-        #{snippet.execute(record[".content"]).join(separator)}
-      </p>
-EOS
-  end
-
-  def render_pagination(request, response, page, limit, total_records)
-    _query = query(request)
-    return if _query.empty?
-
-    return if total_records < limit
-
-    last_page = (total_records / limit.to_f).ceil
-    response.write("<div class='pagination'>\n")
-    if page > 0
-      render_pagination_link(request, response, _query, page - 1, "<<")
-    end
-    last_page.times do |i|
-      if i == page
-        response.write(pagination_span(escape_html(i)))
-      else
-        render_pagination_link(request, response, _query, i, i)
-      end
-    end
-    if page < (last_page - 1)
-      render_pagination_link(request, response, _query, page + 1, ">>")
-    end
-    response.write("</div>\n")
-  end
-
-  def render_pagination_link(request, response, query, page, label)
-    href = "./?query=#{escape_html(query)};page=#{escape_html(page)}"
-    response.write(pagination_span("<a href='#{href}'>#{label}</a>"))
-  end
-
-  def pagination_span(content)
-    "<span class='pagination-link'>#{content}</span>\n"
-  end
-end
-
-run Searcher.new