milkode 0.2.4 → 0.2.9

data/lib/milkode/grep/cli_grep.rb

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+
+require 'optparse'
+require 'milkode/findgrep/findgrep'
+require 'milkode/common/dbdir'
+require 'milkode/cdstk/cdstk_yaml'
+require 'milkode/cdstk/cdstk'
+
+module Milkode
+  class CLI_Grep
+    def self.execute(stdout, arguments=[])
+      option = FindGrep::FindGrep::DEFAULT_OPTION.dup
+      option.dbFile = Dbdir.groonga_path(Dbdir.default_dir)
+      option.isSilent = true
+      
+      my_option = {}
+      my_option[:packages] = []
+
+      begin
+        current_dir = package_root_dir(File.expand_path("."))
+      rescue NotFoundPackage => e
+        current_dir = File.expand_path(".")
+      end
+      
+      # opt = OptionParser.new "#{File.basename($0)} [option] pattern"
+      opt = OptionParser.new "gmilk [option] pattern" # @memo milk grep からも呼ばれるため
+      opt.on('-f KEYWORD', '--file-keyword KEYWORD', 'File path. (Enable multiple call)') {|v| option.filePatterns << v; my_option[:find_mode] = true }
+      opt.on('-d DIR', '--directory DIR', 'Start directory. (deafult:".")') {|v| current_dir = File.expand_path(v); my_option[:find_mode] = true} 
+      opt.on('-s SUFFIX', '--suffix SUFFIX', 'Suffix.') {|v| option.suffixs << v; my_option[:find_mode] = true } 
+      opt.on('-r', '--root', 'Search from package root.') {|v| current_dir = package_root_dir(File.expand_path(".")) }
+      opt.on('-p PACKAGE', '--package PACKAGE', 'Specify search package.') {|v| setup_package(option, my_option, v) }
+      opt.on('-a', '--all', 'Search all package.') {|v| my_option[:all] = true }
+      opt.on('-n NUM', 'Limits the number of match to show.') {|v| option.matchCountLimit = v.to_i }
+      opt.on('-i', '--ignore', 'Ignore case.') {|v| option.ignoreCase = true}
+      opt.on('--color', 'Color highlight.') {|v| option.colorHighlight = true}
+      opt.on('--no-snip', 'There being a long line, it does not snip.') {|v| option.noSnip = true }
+      opt.on('--cache', 'Search only db.') {|v| option.groongaOnly = true }
+      opt.on('--verbose', 'Set the verbose level of output.') {|v| option.isSilent = false }
+      opt.on('-u', '--update', 'With update db.') {|v| my_option[:update] = true }
+      begin
+        opt.parse!(arguments)
+      rescue NotFoundPackage => e
+        stdout.puts "fatal: Not found package '#{e}'."
+        return
+      end
+
+      if option.packages.empty? && !my_option[:all]
+        if (package_dir_in? current_dir)
+          option.filePatterns << current_dir
+        else
+          stdout.puts "fatal: Not package dir '#{current_dir}'."
+          return 
+        end
+      end
+
+      if (arguments.size > 0 || my_option[:find_mode])
+        # update
+        if my_option[:update]
+          cdstk = Cdstk.new(stdout, Dbdir.select_dbdir)
+
+          if (my_option[:all])
+            cdstk.update_all
+          elsif (my_option[:packages].empty?)
+            cdstk.update_package(package_root_dir(File.expand_path(".")))
+          else
+            my_option[:packages].each do |v|
+              cdstk.update_package(v)
+            end
+          end
+          
+          stdout.puts
+        end
+
+        # findgrep
+        findGrep = FindGrep::FindGrep.new(arguments, option)
+        findGrep.searchAndPrint(stdout)
+      else
+        stdout.print opt.help
+      end
+    end
+
+    private
+
+    def self.setup_package(option, my_option, keyword)
+      packages = yaml_load.list( CdstkYaml::Query.new([keyword]) ).map{|v| v['directory']}
+      raise NotFoundPackage.new keyword if (packages.empty?)
+      option.packages += packages
+      my_option[:packages] += packages
+    end
+
+    def self.package_dir_in?(dir)
+      yaml_load.package_root_dir(dir)
+    end
+
+    def self.package_root_dir(dir)
+      package_root = yaml_load.package_root_dir(dir)
+
+      if (package_root)
+        package_root
+      else
+        raise NotFoundPackage.new dir
+      end
+    end
+
+    def self.yaml_load
+      CdstkYaml.load(Dbdir.select_dbdir)
+    end
+
+    # --- error ---
+    class NotFoundPackage < RuntimeError ; end
+
+  end
+end

data/milkode.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{milkode}
-  s.version = "0.2.4"
+  s.version = "0.2.9"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["ongaeshi"]
-  s.date = %q{2011-10-02}
+  s.date = %q{2011-12-24}
   s.description = %q{Line based local source code search engine & web-app.}
   s.email = %q{ongaeshi0621@gmail.com}
-  s.executables = ["cdv", "milk", "cdview"]
+  s.executables = ["cdv", "gmilk", "milk", "cdview"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.rdoc"
@@ -29,8 +29,10 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/cdv",
     "bin/cdview",
+    "bin/gmilk",
     "bin/milk",
     "lib/milkode/cdstk/cdstk.rb",
+    "lib/milkode/cdstk/cdstk_command.rb",
     "lib/milkode/cdstk/cdstk_yaml.rb",
     "lib/milkode/cdstk/cli_cdstk.rb",
     "lib/milkode/cdstk/cli_cdstksub.rb",
@@ -43,6 +45,7 @@ Gem::Specification.new do |s|
     "lib/milkode/cdweb/lib/database.rb",
     "lib/milkode/cdweb/lib/grep.rb",
     "lib/milkode/cdweb/lib/mkurl.rb",
+    "lib/milkode/cdweb/lib/my_nokogiri.rb",
     "lib/milkode/cdweb/lib/query.rb",
     "lib/milkode/cdweb/lib/search_contents.rb",
     "lib/milkode/cdweb/lib/search_files.rb",
@@ -72,17 +75,26 @@ Gem::Specification.new do |s|
     "lib/milkode/common/util.rb",
     "lib/milkode/findgrep/findgrep.rb",
     "lib/milkode/findgrep/result.rb",
+    "lib/milkode/grep/cli_grep.rb",
     "milkode.gemspec",
+    "test/data/a_project/cdstk.rb",
+    "test/data/a_project/cdstk_yaml.rb",
     "test/data/abc.zip",
+    "test/data/b_project/runner.rb",
+    "test/data/b_project/test_dir.rb",
     "test/data/nodir_abc.zip",
     "test/data/nodir_abc_xpi.xpi",
     "test/file_assert.rb",
     "test/file_test_utils.rb",
+    "test/milkode_test_work.rb",
     "test/rake_test_loader.rb",
     "test/runner.rb",
     "test/test_bin_exec.rb",
     "test/test_cdstk.rb",
+    "test/test_cdstk_command.rb",
     "test/test_cdstk_yaml.rb",
+    "test/test_cli_cdstk.rb",
+    "test/test_cli_grep.rb",
     "test/test_coderay_wrapper.rb",
     "test/test_coderay_wrapper_data.rb",
     "test/test_database.rb",
@@ -101,13 +113,21 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.6}
   s.summary = %q{Line based local source code search engine & web-app.}
   s.test_files = [
+    "test/data/a_project/cdstk.rb",
+    "test/data/a_project/cdstk_yaml.rb",
+    "test/data/b_project/runner.rb",
+    "test/data/b_project/test_dir.rb",
     "test/file_assert.rb",
     "test/file_test_utils.rb",
+    "test/milkode_test_work.rb",
     "test/rake_test_loader.rb",
     "test/runner.rb",
     "test/test_bin_exec.rb",
     "test/test_cdstk.rb",
+    "test/test_cdstk_command.rb",
     "test/test_cdstk_yaml.rb",
+    "test/test_cli_cdstk.rb",
+    "test/test_cli_grep.rb",
     "test/test_coderay_wrapper.rb",
     "test/test_coderay_wrapper_data.rb",
     "test/test_database.rb",
@@ -130,8 +150,8 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_runtime_dependency(%q<termcolor>, [">= 1.2.0"])
-      s.add_runtime_dependency(%q<rroonga>, [">= 1.0.0"])
-      s.add_runtime_dependency(%q<rack>, [">= 1.2.1"])
+      s.add_runtime_dependency(%q<rroonga>, [">= 1.1.0"])
+      s.add_runtime_dependency(%q<rack>, [">= 1.3.4"])
       s.add_runtime_dependency(%q<sinatra>, [">= 1.2.6"])
       s.add_runtime_dependency(%q<launchy>, [">= 0.3.7"])
       s.add_runtime_dependency(%q<coderay>, ["= 0.9.8"])
@@ -140,13 +160,14 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<haml>, [">= 3.1.2"])
       s.add_runtime_dependency(%q<sass>, [">= 3.1.3"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0.8.2"])
     else
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<termcolor>, [">= 1.2.0"])
-      s.add_dependency(%q<rroonga>, [">= 1.0.0"])
-      s.add_dependency(%q<rack>, [">= 1.2.1"])
+      s.add_dependency(%q<rroonga>, [">= 1.1.0"])
+      s.add_dependency(%q<rack>, [">= 1.3.4"])
       s.add_dependency(%q<sinatra>, [">= 1.2.6"])
       s.add_dependency(%q<launchy>, [">= 0.3.7"])
       s.add_dependency(%q<coderay>, ["= 0.9.8"])
@@ -155,14 +176,15 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<haml>, [">= 3.1.2"])
       s.add_dependency(%q<sass>, [">= 3.1.3"])
       s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+      s.add_dependency(%q<hpricot>, [">= 0.8.2"])
     end
   else
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<termcolor>, [">= 1.2.0"])
-    s.add_dependency(%q<rroonga>, [">= 1.0.0"])
-    s.add_dependency(%q<rack>, [">= 1.2.1"])
+    s.add_dependency(%q<rroonga>, [">= 1.1.0"])
+    s.add_dependency(%q<rack>, [">= 1.3.4"])
     s.add_dependency(%q<sinatra>, [">= 1.2.6"])
     s.add_dependency(%q<launchy>, [">= 0.3.7"])
     s.add_dependency(%q<coderay>, ["= 0.9.8"])
@@ -171,6 +193,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<haml>, [">= 3.1.2"])
     s.add_dependency(%q<sass>, [">= 3.1.3"])
     s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+    s.add_dependency(%q<hpricot>, [">= 0.8.2"])
   end
 end
 

data/test/data/a_project/cdstk.rb

@@ -0,0 +1,634 @@
+# -*- coding: utf-8 -*-
+
+require 'yaml'
+require 'pathname'
+require 'rubygems'
+require 'groonga'
+require 'fileutils'
+require 'pathname'
+require 'milkode/cdstk/cdstk_yaml'
+require 'milkode/common/grenfiletest'
+require 'milkode/common/util'
+require 'milkode/common/dir'
+include Milkode
+require 'kconv'
+begin
+  require 'readline'
+rescue LoadError
+  $no_readline = true
+end
+require 'milkode/cdweb/lib/database'
+require 'open-uri'
+require 'milkode/cdstk/cdstk_command'
+
+module Milkode
+  class Cdstk
+    # バイグラムでトークナイズする。連続する記号・アルファベット・数字は一語として扱う。
+    # DEFAULT_TOKENIZER = "TokenBigram"
+
+    # 記号・アルファベット・数字もバイグラムでトークナイズする。
+    DEFAULT_TOKENIZER = "TokenBigramSplitSymbolAlphaDigit"
+
+    class ConvetError < RuntimeError ; end
+    
+    def initialize(io = $stdout, db_dir = ".")
+      @db_dir = db_dir
+      Database.setup(@db_dir)
+      @out = io
+      # @out = $stdout # 強制出力
+      clear_count
+    end
+
+    def clear_count
+      @package_count = 0
+      @file_count = 0
+      @add_count = 0
+      @update_count = 0
+      @start_time = Time.now
+    end
+
+    def init
+      if Dir.emptydir?(@db_dir)
+        CdstkYaml.create(@db_dir)
+        @out.puts "create     : #{yaml_file}"
+        db_create(db_file)
+      else
+        @out.puts "Can't create milkode db (Because not empty in #{db_dir_expand})"
+      end
+    end
+
+    def compatible?
+      db_open(db_file)
+    end
+
+    def update_all
+      print_result do 
+        yaml = yaml_load
+
+        db_open(db_file)
+
+        yaml.list.each do |content|
+          update_dir_in(content["directory"])
+        end
+      end
+    end
+
+    def update(args, options)
+      if (options[:all])
+        update_all
+      else
+        if (args.empty?)
+          path = File.expand_path('.')
+          package = yaml_load.package_root( path )
+
+          if (package)
+            print_result do
+              db_open(db_file)
+              update_dir_in(package["directory"])
+            end
+          else
+            @out.puts "Not registered. If you want to add, 'milk add #{path}'."
+          end
+        else
+          print_result do
+            update_list = yaml_load.list CdstkYaml::Query.new(args)
+            db_open(db_file)
+            update_list.each do |content|
+              update_dir_in(content["directory"])
+            end
+          end
+        end
+      end
+    end
+
+    def update_package(dir)
+      db_open(db_file)
+      update_dir(dir)
+    end
+
+    def update_dir(dir)
+      update_dir_in(dir)
+    end
+
+    def add(contents)
+      # YAMLを読み込み
+      yaml = yaml_load
+
+      # コンテンツを読み込める形に変換
+      begin
+        contents.map!{|v|convert_content(v)}
+      rescue ConvetError
+        return
+      end
+
+      # 存在しないコンテンツがあった場合はその場で終了
+      contents.each do |v|
+        shortname = File.basename v
+
+        if (yaml.cant_add_directory? v)
+          error_alert("already exist '#{shortname}'.")
+          return
+        end
+          
+        unless (File.exist? v)
+          error_alert("not found '#{v}'.")
+          return
+        end
+      end
+
+      # YAML更新
+      yaml.add(contents)
+      yaml.save
+
+      # 部分アップデート
+      print_result do 
+        db_open(db_file)
+        contents.each do |dir|
+          update_dir(dir)
+        end
+      end
+    end
+
+    def convert_content(src)
+      # httpファイルならダウンロード
+      begin
+        src = download_file(src)
+      rescue => e
+        error_alert("download failure '#{src}'.")
+        raise e                 # そのまま上に持ち上げてスタックトレース表示
+      end
+      
+      # アーカイブファイルなら展開
+      begin
+        src = extract_file(src)
+      rescue => e
+        error_alert("extract failure '#{src}'.")
+        raise e                 # そのまま上に持ち上げてスタックトレース表示
+      end
+
+      # 絶対パスに変換
+      File.expand_path(src)
+    end
+
+    def extract_file(src)
+      ext = File.extname(src);
+      
+      case ext
+      when '.zip', '.xpi'
+        alert("extract", "#{src}")
+        zip_dir = File.join(@db_dir, "packages/#{ext.sub(".", "")}")
+        result = File.join(zip_dir, Util::zip_extract(src, zip_dir))
+      else
+        src
+      end
+    end
+
+    def download_file(src)
+      if (src =~ /^https?:/)
+        download_file_in(src)
+      else
+        src
+      end
+    end
+
+    def download_file_in(url)
+      alert("download", "#{url}")
+      
+      dst_dir = File.join(@db_dir, "packages/http")
+      FileUtils.mkdir_p dst_dir
+
+      filename = File.join(dst_dir, File.basename(url))
+      
+      open(url) do |src|
+        open(filename, "wb") do |dst|
+          dst.write(src.read)
+        end
+      end
+
+      filename
+    end
+
+    def remove(args, options)
+      print_result do 
+        db_open(db_file)
+        
+        yaml = yaml_load
+        query = CdstkYaml::Query.new(args)
+        
+        remove_list = yaml_load.list(query)
+        return if remove_list.empty?
+        
+        list(args, {:verbose => true})
+        
+        if options[:force] or yes_or_no("Remove #{remove_list.size} contents? (yes/no)")
+          # yamlから削除
+          yaml.remove(query)
+          yaml.save
+          
+          # データベースからも削除
+          packages = remove_list.map{|v| File.basename v['directory']}
+
+          # 本当はパッケージの配列をまとめて渡した方が効率が良いのだが、表示を綺麗にするため
+          packages.each do |package|
+            alert("rm_package", package)
+            @package_count += 1
+            
+            Database.instance.remove([package]) do |record|
+              alert("rm_record", record.path)
+              @file_count += 1
+            end
+          end
+        end
+      end
+    end
+
+    def yes_or_no(msg)
+      if ($no_readline)
+        @out.puts "Pass Cdstk#yes_or_no, because fail \"require 'readline'\"."
+        return true
+      end
+        
+      @out.puts msg
+      while buf = Readline.readline("> ")
+        case buf
+        when 'yes'
+          return true
+        when 'no'
+          break
+        end
+      end
+      return false
+    end
+
+    def list(args, options)
+      query = (args.empty?) ? nil : CdstkYaml::Query.new(args)
+      a = yaml_load.list(query).map {|v| [File.basename(v['directory']), v['directory']] }
+      max = a.map{|v|v[0].length}.max
+      str = a.sort_by {|v|
+        v[0]
+      }.map {|v|
+        h = File.exist?(v[1]) ? '' : '? '
+        if (options[:verbose])
+          "#{(h + v[0]).ljust(max+2)} #{v[1]}"
+        else
+          "#{h}#{v[0]}"
+        end
+      }.join("\n")
+
+      @out.puts str
+    end
+
+    def pwd(options)
+      dir = options[:default] ? Dbdir.default_dir : db_dir_expand
+      
+      if File.exist? dir
+        @out.puts dir
+      else
+        @out.puts "Not found db in #{Dir.pwd}"
+      end
+    end
+
+    def cleanup(options)
+      # 互換性テスト
+      db_open(db_file)
+
+      # cleanup開始
+      if (options[:force] or yes_or_no("cleanup contents? (yes/no)"))
+        print_result do 
+          # yamlファイルのクリーンアップ
+          yaml = yaml_load
+          
+          yaml.cleanup do |v|
+            alert("rm_package", v['directory'])
+            @package_count += 1
+          end
+          
+          yaml.save
+          
+          # データベースのクリーンアップ
+          Database.instance.cleanup do |record|
+            alert("rm_record", record.path)
+            @file_count += 1
+          end
+        end
+      end
+    end
+
+    def rebuild
+      db_delete(db_file)
+      db_create(db_file)
+      update_all
+    end
+
+    def dump
+      db_open(db_file)
+      
+      documents = Groonga::Context.default["documents"]
+      records = documents.select
+      records.each do |record|
+        @out.puts record.inspect
+        @out.puts "path : #{record.path}"
+        @out.puts "shortpath : #{record.shortpath}"
+        @out.puts "suffix : #{record.suffix}"
+        @out.puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
+        @out.puts "content :", record.content ? record.content[0..64] : nil
+        @out.puts
+      end
+      @out.puts "total #{records.size} record."
+    end
+
+    def dir(args, options)
+      yaml = yaml_load
+      
+      if args.empty?
+        path = File.expand_path('.')
+        package = yaml.package_root(path)
+
+        if (package)
+          @out.puts package['directory']
+        else
+          @out.puts "Not registered. If you want to add, 'milk add #{path}'."
+        end
+      else
+        @out.puts yaml.list(CdstkYaml::Query.new(args)).map{|v|v['directory']}
+      end
+    end
+
+    def setdb(args, options)
+      if (options[:reset])
+        CdstkCommand.setdb_reset
+        @out.puts "Reset default db\n  remove:      #{Dbdir.milkode_db_dir}\n  default_db:  #{Dbdir.default_dir}"
+      elsif (args.empty?)
+        @out.puts Dbdir.default_dir
+      else
+        path = File.expand_path(args[0])
+        begin
+          CdstkCommand.setdb_set path
+          @out.puts "Set default db #{path}."
+        rescue CdstkCommand::NotExistDatabase
+          @out.puts "fatal: '#{path}' is not database."
+        end
+      end
+    end
+
+    def mcd(args, options)
+      @out.print <<EOF
+# Copy to '.bashrc'.
+mcd() {
+    cd `milk dir $1 $2 $3 $4 $5 $6 $7 $8 $9`; pwd
+}
+EOF
+    end
+
+    private
+
+    def db_file
+      Dbdir.expand_groonga_path(@db_dir)
+    end
+
+    def db_file_expand
+      File.expand_path(db_file)
+    end
+
+    def db_dir_expand
+      File.expand_path(@db_dir)
+    end
+
+    def custom_db?
+      db_dir_expand != Dbdir.default_dir
+    end
+
+    def yaml_file
+      CdstkYaml.yaml_file @db_dir
+    end
+
+    def yaml_load
+      CdstkYaml.load(@db_dir)
+    end
+
+    def update_dir_in(dir)
+      alert("package", File.basename(dir) )
+      @package_count += 1
+      
+      dir = File.expand_path(dir)
+
+      if (!FileTest.exist?(dir))
+        @out.puts "[WARNING]  : #{dir} (Not found, skip)"
+      elsif (FileTest.directory? dir)
+        db_add_dir(dir)
+      else
+        db_add_file(STDOUT, dir, File.basename(dir))
+      end
+    end
+
+    def time
+      @end_time - @start_time 
+    end
+
+    def print_result
+      clear_count
+      yield
+      @end_time = Time.now
+      
+      result_info
+      milkode_info
+    end
+
+    def result_info
+      r = []
+      r << "#{@package_count} packages" if @package_count > 0
+      r << "#{@file_count} records" if @file_count > 0
+      r << "#{@add_count} add" if @add_count > 0
+      r << "#{@update_count} update" if @update_count > 0
+      r.join(', ')
+      alert('result', "#{r.join(', ')}. (#{Gren::Util::time_s(time)})")
+    end
+
+    def milkode_info
+      alert('*milkode*', "#{yaml_load.package_num} packages, #{Database.instance.totalRecords} records in #{db_file}.")
+    end
+
+    def db_create(filename)
+      dbfile = Pathname(File.expand_path(filename))
+      dbdir = dbfile.dirname
+      dbdir.mkpath unless dbdir.exist?
+      
+      unless dbfile.exist?
+        Groonga::Database.create(:path => dbfile.to_s)
+        db_define
+        @out.puts "create     : #{filename} created."
+      else
+        @out.puts "message    : #{filename} already exist."
+      end
+    end
+
+    def db_define
+      Groonga::Schema.define do |schema|
+        schema.create_table("documents", :type => :hash) do |table|          
+          table.string("path")
+          table.string("shortpath")
+          table.text("content")
+          table.time("timestamp")
+          table.text("suffix")
+        end
+
+        schema.create_table("terms",
+                            :type => :patricia_trie,
+                            :key_normalize => true,
+                            :default_tokenizer => DEFAULT_TOKENIZER) do |table|
+          table.index("documents.path", :with_position => true)
+          table.index("documents.shortpath", :with_position => true)
+          table.index("documents.content", :with_position => true)
+          table.index("documents.suffix", :with_position => true)
+        end
+      end
+    end
+
+    def db_open(filename)
+      dbfile = Pathname(File.expand_path(filename))
+      
+      if dbfile.exist?
+        # データベースを開く
+        Groonga::Database.open(dbfile.to_s)
+
+        # 互換性テスト
+        db_compatible?
+      else
+        raise "error      : #{dbfile.to_s} not found!!"
+      end
+    end
+
+    def db_delete(filename)
+      raise "Illegal file name : #{filename}." unless filename =~ /\.db$/
+      Dir.glob("#{filename}*").each do |f|
+        @out.puts "delete     : #{f}"
+        FileUtils.rm_r(f)
+      end
+    end
+    private :db_delete
+      
+    def db_add_dir(dirname)
+      searchDirectory(STDOUT, dirname, File.basename(dirname), 0)
+    end
+    private :db_add_dir
+
+    def db_add_file(stdout, filename, shortpath)
+      # ファイル名を全てUTF-8に変換
+      filename_utf8 = Util::filename_to_utf8(filename)
+      shortpath_utf8 = Util::filename_to_utf8(shortpath)
+      suffix_utf8 = File::extname(filename_utf8)
+      
+      # 格納するデータ
+      values = {
+        :path => filename_utf8,
+        :shortpath => shortpath_utf8,
+        :content => nil,
+        :timestamp => File.mtime(filename),
+        :suffix => suffix_utf8,
+      }
+      
+      # 検索するデータベース
+      documents = Groonga::Context.default["documents"]
+      
+      record = documents[ values[:path] ]
+      isNewFile = false
+
+      unless record
+        document = documents.add(values[:path])
+        isNewFile = true
+      else
+        document = record
+      end
+      
+      # タイムスタンプが新しければデータベースに格納
+      if (document[:timestamp] < values[:timestamp])
+        # 実際に使うタイミングでファイルの内容を読み込み
+        # values[:content] = open(filename).read
+        # データベース内の文字コードは'utf-8'で統一
+        values[:content] = Kconv.kconv(File.read(filename), Kconv::UTF8)
+        
+        # データベースに格納
+        values.each do |key, value|
+          if (key == :path)
+            if (isNewFile)
+              @add_count += 1
+              alert("add_record", value)
+            else
+              @update_count += 1
+              alert("update", value)
+            end
+          end
+          document[key] = value
+        end
+      end
+
+    end
+
+    def searchDirectory(stdout, dir, shortdir, depth)
+      Dir.foreach(dir) do |name|
+        next if (name == '.' || name == '..')
+          
+        fpath = File.join(dir,name)
+        shortpath = File.join(shortdir,name)
+        
+        # 除外ディレクトリならばパス
+        next if ignoreDir?(fpath)
+
+        # 読み込み不可ならばパス
+        next unless FileTest.readable?(fpath)
+
+        # ファイルならば中身を探索、ディレクトリならば再帰
+        case File.ftype(fpath)
+        when "directory"
+          searchDirectory(stdout, fpath, shortpath, depth + 1)
+        when "file"
+          unless ignoreFile?(fpath)
+            db_add_file(stdout, fpath, shortpath)
+            @file_count += 1
+            # @out.puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
+          end
+        end          
+      end
+    end
+
+    def ignoreDir?(fpath)
+      FileTest.directory?(fpath) &&
+      GrenFileTest::ignoreDir?(fpath)
+    end
+    private :ignoreDir?
+
+    def ignoreFile?(fpath)
+      GrenFileTest::ignoreFile?(fpath) ||
+      GrenFileTest::binary?(fpath)
+    end
+    private :ignoreFile?
+
+    def alert(title, msg)
+      if (Util::platform_win?)
+        @out.puts "#{title.ljust(10)} : #{Kconv.kconv(msg, Kconv::SJIS)}"
+      else
+        @out.puts "#{title.ljust(10)} : #{msg}"
+      end
+    end
+
+    def error_alert(msg)
+      @out.puts "[fatal] #{msg}"
+    end
+
+    def db_compatible?
+      begin
+        db_define
+      rescue Groonga::Schema::Error => e
+        puts <<EOF
+Milkode repository is old -> #{db_dir_expand}.
+Please rebuild repository, 
+
+  milk rebuild
+
+See 'milk --help' or http://milkode.ongaeshi.me .
+EOF
+        exit -1
+      end
+    end
+  end
+end