rroonga 1.0.8 → 1.0.9

data/NEWS.ja.rdoc

@@ -1,5 +1,52 @@
 = お知らせ
 
+== 1.0.9: 2011-01-29
+
+=== 改良
+
+* Windows上でのgem作成に対応。
+  [ongaeshiさんがパッチ提供]
+* Groonga::Schemaでテーブル・カラムを削除した時に作成したディレクトリを
+  削除するようにした。
+* Groonga::Context#create_databaseを追加。
+* Groonga::Context#open_databaseを追加。
+* Groonga::Column#indexesを追加。
+* Groonga::Table#select内でのインデックスカラムにマッチする記法をサポート。
+    table.select do |record|
+      record.match("query") do |match_record|
+        (match_record.index("Terms.title") * 1000) |
+          (match_record.index("Terms.description") * 100)
+          match_record.content
+      end
+    end
+* Groonga::Table#select内での前方一致検索に対応。
+    table.select do |record|
+      record.name.prefix_search("groo")
+    end
+* Groonga::Table#select内での後方一致検索に対応。
+    table.select do |record|
+      record.name.suffix_search("nga")
+    end
+* :default_tokenizerのスキーマダンプに対応。
+* :key_normalizeのスキーマダンプに対応。
+* Groonga::Table#have_column?が擬似カラムに対応。
+* Groonga::Record#have_column?が擬似カラムに対応。
+
+=== 変更
+
+* Groonga::OperatoionをGroonga::Operatorに変更。
+  （しばらくはGroonga::Operationも使えるが、非推奨。）
+
+=== 修正
+
+* 複数のGroonga::Contextを利用した場合に
+  Groonga::Table#selectでクラッシュする問題を修正。
+* 例外発生時にクラッシュする問題を修正。
+
+=== 感謝
+
+* ongaeshioさん
+
 == 1.0.8: 2010-12-25
 
 === 改良

data/NEWS.rdoc

@@ -1,5 +1,53 @@
 = NEWS
 
+== 1.0.9: 2011-01-29
+
+=== Improvements
+
+* Supported gem creation on Windows.
+  [Patch by ongaeshi]
+* Supported generated directory that is created by Groonga::Schema removal
+  when table or column is removed.
+* Added Groonga::Context#create_database.
+* Added Groonga::Context#open_database.
+* Added Groonga::Column#indexes.
+* Supported a notation for specifying index column as match target in
+  Groonga::Table#select:
+    table.select do |record|
+      record.match("query") do |match_record|
+        (match_record.index("Terms.title") * 1000) |
+          (match_record.index("Terms.description") * 100)
+          match_record.content
+      end
+    end
+* Supported prefix search in Groonga::Table#select:
+    table.select do |record|
+      record.name.prefix_search("groo")
+    end
+* Supported suffix search in Groonga::Table#select:
+    table.select do |record|
+      record.name.suffix_search("nga")
+    end
+* Supported :default_tokenizer schema dump.
+* Supported :key_normalize schema dump.
+* Supported pseudo columns by Groonga::Table#have_column?.
+* Supported pseudo columns by Groonga::Record#have_column?.
+
+=== Changes
+
+* Renamed Groonga::Operatoion to Groonga::Operator.
+  (Groonga::Operation is deprecated but still usable.)
+
+=== Fixes
+
+* Fixed a crash bug when not default Groonga::Context is used in
+  Groonga::Table#select.
+* Fixed a crash bug when an exception is occurred.
+
+=== Thanks
+
+* ongaeshio
+
 == 1.0.8: 2010-12-25
 
 === Improvements

data/README.ja.rdoc

@@ -65,3 +65,4 @@ http://lists.sourceforge.jp/mailman/listinfo/groonga-dev
 * 咳さん: バグレポートしてくれました。
 * 小野マトペさん: バグレポートしてくれました。
 * @kamipoさん: バグレポートしてくれました。
+* ongaeshiさん: Windows上でgemをビルドするためのパッチをくれました。

data/README.rdoc

@@ -65,3 +65,4 @@ http://rubyforge.org/mailman/listinfo/groonga-users-en
 * m_seki: sent bug reports.
 * ono matope: sent bug reports.
 * @kamipo: send a bug report.
+* ongaeshi: sent a patch to build gem on Windows.

data/Rakefile

@@ -144,28 +144,41 @@ ObjectSpace.each_object(Rake::RDocTask) do |rdoc_task|
   rdoc_task.rdoc_files += Dir.glob("**/*.rdoc")
 end
 
+def windows?(platform=nil)
+  platform ||= RUBY_PLATFORM
+  platform =~ /mswin(?!ce)|mingw|cygwin|bccwin/
+end
+
+def collect_binary_files(binary_dir)
+  binary_files = []
+  Find.find(binary_dir) do |name|
+    next unless File.file?(name)
+    next if /\.zip\z/i =~ name
+    binary_files << name
+  end
+  binary_files
+end
+
 relative_vendor_dir = "vendor"
 relative_binary_dir = File.join("vendor", "local")
 vendor_dir = File.join(base_dir, relative_vendor_dir)
 binary_dir = File.join(base_dir, relative_binary_dir)
+
 Rake::ExtensionTask.new("groonga", project.spec) do |ext|
-  ext.cross_compile = true
-  ext.cross_compiling do |spec|
-    if /mingw|mswin/ =~ spec.platform.to_s
-      binary_files = []
-      Find.find(relative_binary_dir) do |name|
-        next unless File.file?(name)
-        next if /\.zip\z/i =~ name
-        binary_files << name
+  if windows?
+    ext.gem_spec.files += collect_binary_files(relative_binary_dir)
+  else
+    ext.cross_compile = true
+    ext.cross_compiling do |spec|
+      if windows?(spec.platform.to_s)
+        spec.files += collect_binary_files(relative_binary_dir)
       end
-      spec.files += binary_files
     end
   end
 end
 
 task :publish_docs => [:prepare_docs_for_publishing]
 
-
 include ERB::Util
 
 def apply_template(file, head, header, footer)
@@ -236,8 +249,14 @@ project.spec.executables.clear
 task(:release).prerequisites.reject! {|name| name == "clean"}
 
 namespace :win32 do
+  if ENV["GROONGA64"] == "yes"
+    host = "amd64-mingw32msvc"
+  else
+    host = "i586-mingw32msvc"
+  end
+
   desc "Build MeCab and groonga and install them into vendor/local/."
-  task(:build => :build_groonga)
+  task(:build => [:build_mecab, :build_mecab_dict, :build_groonga])
 
   desc "Build MeCab and install it into vendor/local/."
   task(:build_mecab) do
@@ -260,7 +279,7 @@ namespace :win32 do
     Dir.chdir(File.join(tmp_dir, mecab_base)) do
       sh("./configure",
          "--prefix=#{binary_dir}",
-         "--host=amd64-mingw32msvc") or exit(false)
+         "--host=#{host}") or exit(false)
       sh("env", "GREP_OPTIONS=--text", "nice", "make", "-j8") or exit(false)
       sh("env", "GREP_OPTIONS=--text", "make", "install") or exit(false)
 
@@ -321,7 +340,7 @@ namespace :win32 do
       mecab_config = File.join(binary_dir, "bin", "mecab-config")
       args = ["./configure",
               "--prefix=#{binary_dir}",
-              "--host=amd64-mingw32msvc",
+              "--host=#{host}",
               "--without-cutter",
               "--disable-benchmark"]
       if File.exist?(mecab_config)

data/benchmark/create-wikipedia-database.rb

@@ -0,0 +1,212 @@
+# Wikipedia data: http://download.wikimedia.org/jawiki/latest/jawiki-latest-pages-articles.xml.bz2
+
+require 'time'
+require 'fileutils'
+require 'groonga'
+
+require 'nokogiri'
+
+class WikipediaExtractor
+  def initialize(listener)
+    @listener = listener
+  end
+
+  def extract(input)
+    extractor = Extractor.new(@listener)
+    parser = Nokogiri::XML::SAX::Parser.new(extractor)
+    parser.parse(input)
+  end
+
+  class Extractor
+    def initialize(listener)
+      @listener = listener
+      @name_stack = []
+      @text_stack = []
+      @contributor_stack = []
+      @page = {}
+    end
+
+    def start_document
+    end
+
+    def end_document
+    end
+
+    def start_element_namespace(name, attrs=[], prefix=nil, uri=nil, ns=[])
+      @name_stack << name
+      @text_stack << ""
+      case @name_stack.join(".")
+      when "mediawiki.page"
+        @page = {}
+      when "mediawiki.page.revision.contributor"
+        @contributor_stack << {}
+      end
+    end
+
+    def end_element_namespace(name, prefix=nil, uri=nil)
+      case @name_stack.join(".")
+      when "mediawiki.page"
+        @listener.page(@page)
+      when "mediawiki.page.title"
+        title = @text_stack.last
+        @page[:title] = @listener.title(title) || title
+      when "mediawiki.page.revision.timestamp"
+        timestamp = Time.parse(@text_stack.last)
+        @page[:timestamp] = @listener.timestamp(timestamp) || timestamp
+      when "mediawiki.page.revision.contributor"
+        contributor = @contributor_stack.pop
+        @page[:contributor] = @listener.contributor(contributor) || contributor
+      when "mediawiki.page.revision.contributor.id"
+        @contributor_stack.last[:id] = Integer(@text_stack.last)
+      when "mediawiki.page.revision.contributor.username"
+        @contributor_stack.last[:name] = @text_stack.last
+      when "mediawiki.page.revision.text"
+        content = @text_stack.last
+        @page[:content] = @listener.content(content) || content
+      end
+      @name_stack.pop
+      @text_stack.pop
+    end
+
+    def characters(string)
+      elements_without_interested_text = [
+                                          "mediawiki", "siteinfo", "case",
+                                          "namespaces", "revisions",
+                                          "contributor",
+                                         ]
+      return if elements_without_interested_text.include?(@name_stack.last)
+      @text_stack.last << string
+    end
+
+    def xmldecl(*arguments, &block)
+    end
+
+    def comment(string)
+    end
+
+    def warning(string)
+    end
+
+    def error(string)
+    end
+
+    def cdata_block(string)
+    end
+  end
+end
+
+class WikipediaImporter
+  def initialize(groonga_loader)
+    @groonga_loader = groonga_loader
+  end
+
+  def title(title)
+  end
+
+  def timestamp(timestamp)
+  end
+
+  def contributor(contributor)
+  end
+
+  def content(content)
+  end
+
+  def page(page)
+    @groonga_loader.load(page)
+  end
+end
+
+module TimeDrilldownable
+  def define_time_columns(table)
+    table.int32("year")
+    table.int32("month")
+    table.short_text("date")
+    table.int32("wday")
+    table.int32("hour")
+  end
+
+  def add_time(table, key, time)
+    table[key]["year"] = time.year
+    table[key]["month"] = time.month
+    table[key]["date"] = time.strftime("%m/%d")
+    table[key]["wday"] = time.wday
+    table[key]["hour"] = time.hour
+  end
+end
+
+class GroongaLoader
+  include TimeDrilldownable
+
+  def initialize
+    FileUtils.rm_rf("/tmp/wikipedia-db")
+    FileUtils.mkdir_p("/tmp/wikipedia-db")
+    @context = Groonga::Context.new
+    @context.create_database("/tmp/wikipedia-db/db")
+
+    Groonga::Schema.define(:context => @context) do |schema|
+      schema.create_table("Users", :type => :hash, :key_type => "Int64") do |table|
+        table.short_text("name")
+      end
+
+      schema.create_table("Documents", :type => :patricia_trie, :key_type => "ShortText") do |table|
+        table.long_text("content")
+        table.time("timestamp")
+        define_time_columns(table)
+        table.reference("last_contributor", "Users")
+        table.column("links", "Documents", :type => :vector)
+      end
+
+      schema.create_table("Terms", :type => :hash, :default_tokenizer => "TokenBigram") do |table|
+        table.index("Documents._key")
+        table.index("Documents.content")
+      end
+    end
+    @documents = @context["Documents"]
+    @users = @context["Users"]
+    @terms = @context["Terms"]
+  end
+
+  LOCK_TIMEOUT_SECONDS = 10
+  def lock
+    @context.database.lock(:timeout => LOCK_TIMEOUT_SECONDS * 1000) do
+      yield
+    end
+  end
+
+  def load(page)
+    lock do
+      do_load(page)
+    end
+  end
+
+  def do_load(page)
+    content = page.delete(:content)
+    timestamp = page.delete(:timestamp)
+    title = page.delete(:title)
+    contributor = page.delete(:contributor)
+
+    puts "loading: #{title}"
+    @documents.add(title, :content => content, :timestamp => timestamp)
+    load_links(title, content)
+    add_time(@documents, title, timestamp)
+
+    if not contributor.empty?
+      @documents.add(title, :last_contributor => contributor[:id])
+      @users[contributor[:id]][:name] = contributor[:name]
+    end
+  end
+
+  def load_links(title, content)
+    links = content.scan(/\[\[.*?\]\]/)
+    links = links.collect do |link|
+      link.sub(/\A\[\[/, '').sub(/\]\]\z/, '').sub(/\|[^\|]+\z/, '')
+    end
+    @documents.add(title, :links => links)
+  end
+end
+
+if __FILE__ == $0
+  extractor = WikipediaExtractor.new(WikipediaImporter.new(GroongaLoader.new))
+  extractor.extract(ARGF)
+end

data/benchmark/repeat-load.rb

@@ -0,0 +1,213 @@
+require 'fileutils'
+require 'shellwords'
+
+require 'groonga'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'create-wikipedia-database'
+
+class SampleRecords
+  def initialize(record_count)
+    @record_count = record_count
+    @current_count = 0
+    @records = []
+
+    initialize_sample_records
+  end
+
+  def initialize_sample_records
+    extractor = WikipediaExtractor.new(WikipediaImporter.new(self))
+    #@records = @record_count.times.collect do
+    #  create_random_item
+    #end
+    catch(:stop_extract) do
+      extractor.extract(ARGF)
+    end
+  end
+
+  def load(page)
+    @current_count += 1
+    record = {
+      :_key => page[:title],
+      :content => page[:content],
+    }
+    #pp record
+    @records << record
+    throw :stop_extract if @current_count == @record_count
+  end
+
+  def values(count=nil)
+    count ||= @record_count
+
+    if count == 1
+      [first_record]
+    else
+      @records[0, count - 1] + [first_record]
+    end
+  end
+
+  def first_record
+    @records.first
+  end
+
+  def each(*arguments, &block)
+    values.each(*arguments, &block)
+  end
+
+  def n_records(count)
+    values(count)
+  end
+
+  def create_random_item
+    {"_key" => "ryoqun"}
+  end
+end
+
+class RepeatLoadRunner
+  DATABASE_DIRECTORY = "/tmp/repeat-overwrite"
+
+  def initialize(sample_records, options=nil)
+    @options = options || {}
+    @context = Groonga::Context.new(:encoding => :none)
+    @sample_records = sample_records
+  end
+
+  DEFAULT_REPEAT_COUNT = 1
+  DEFAULT_RECORD_COUNT = 1
+  def repeat_count
+    @options[:repeat_count] || DEFAULT_REPEAT_COUNT
+  end
+
+  def record_count
+    @options[:record_count] || DEFAULT_RECORD_COUNT
+  end
+
+  def with_index?
+    @options[:with_index]
+  end
+
+  def database_directory
+    DATABASE_DIRECTORY
+  end
+
+  def database_path
+    "#{database_directory}/db"
+  end
+
+  def setup_database
+    FileUtils.rm_rf(database_directory)
+    FileUtils.mkdir_p(database_directory)
+
+    @context.create_database(database_path)
+    Groonga::Schema.define(:context => @context) do |schema|
+      schema.create_table("Contents", :type => :hash, :key_type => "ShortText") do |table|
+        table.long_text("content")
+      end
+
+      if with_index?
+        schema.create_table("Terms", :type => :hash, :default_tokenizer => "TokenBigram") do |table|
+          table.index("Contents._key")
+          table.index("Contents.content")
+        end
+      end
+    end
+  end
+
+  def run
+    setup_database
+
+    before_load
+    add_record
+    after_load
+  end
+
+  private
+  def add_record
+    puts "loading..."
+    repeat_count.times do |count|
+      add_record_via_load_command
+      if repeat_count != 1 and count.zero?
+        after_first_load
+      end
+    end
+    puts "... (#{repeat_count} times repeated)"
+    puts
+  end
+
+  def add_record_via_load_command
+    puts "iteration: loading #{record_count} records"
+    count = 0
+    @sample_records.n_records(record_count).each do |record|
+      mangle_record(record)
+
+      #pp record
+      command = "load --table Contents --input_type json --values '#{JSON.generate(record).gsub(/\'/, '').gsub(/\"/, '\\\"')}'"
+      #puts command
+      @context.send(command)
+      if record_count != 1 and count.zero?
+        after_first_load
+      end
+      count += 1
+    end
+  end
+
+  def mangle_record(record)
+    record[:content] = record[:content][0, 400]
+    record[:_key] = :FIXED_KEY
+  end
+
+  def before_load
+    puts "before load:"
+    measure_database_size
+    puts
+  end
+
+  def after_first_load
+    puts "after first load:"
+    measure_database_size
+    puts
+  end
+
+  def after_load
+    puts "after load:"
+    measure_database_size
+    puts
+  end
+
+  def measure_database_size
+    #measure_apparent_size
+    measure_actual_size
+  end
+
+  def measure_apparent_size
+    puts "apparent disk usage:"
+    puts execute_du("--apparent-size")
+  end
+
+  def measure_actual_size
+    puts "actual disk usage:"
+    puts execute_du
+  end
+
+  def execute_du(options=nil)
+    `sync`
+    `find #{database_directory} -type f -print0 | xargs -0 du --human-readable #{options.to_s} | sort -k 2 | uniq`
+  end
+end
+
+sample_records = SampleRecords.new(1000)
+
+#puts "load one record, repeat one time"
+#RepeatLoadRunner.new(sample_records).run
+
+#puts "load one record, repeat 100 time"
+#RepeatLoadRunner.new(sample_records, :repeat_count => 100).run
+
+#puts "load one record, repeat 100 time with index column defined"
+#RepeatLoadRunner.new(sample_records, :repeat_count => 100, :with_index => true).run
+
+puts "load 100 records, repeat 1 time"
+RepeatLoadRunner.new(sample_records, :record_count => 100).run
+
+puts "load 100 records, repeat 1 time with index column defined"
+RepeatLoadRunner.new(sample_records, :record_count => 100, :with_index => true).run