codestock 0.1.0

data/lib/mkgrendb/mkgrendb.rb

@@ -0,0 +1,296 @@
+# -*- coding: utf-8 -*-
+
+require 'yaml'
+require 'pathname'
+require 'rubygems'
+require 'groonga'
+require 'fileutils'
+require 'pathname'
+require File.join(File.dirname(__FILE__), 'grendbyaml')
+require File.join(File.dirname(__FILE__), '../common/grenfiletest')
+require File.join(File.dirname(__FILE__), '../common/util')
+include Gren
+
+module Mkgrendb
+  class Mkgrendb
+    DB_FILE_PATH = 'db/grendb.db'
+    
+    def initialize(io = $stdout, db_dir = ".")
+      @db_dir = db_dir
+      @out = io
+      @file_count = 0
+      @add_count = 0
+      @update_count = 0
+      @start_time = Time.now
+    end
+
+    def init
+      if Dir.entries(@db_dir) == [".", ".."]
+        GrendbYAML.create(@db_dir)
+        @out.puts "create     : #{yaml_file}"
+        db_create(db_file)
+      else
+        @out.puts "Can't create Grendb Database (Not empty) in #{@db_dir}"
+      end
+    end
+
+    def update
+      print_result do 
+        yaml = yaml_load
+        db_open(db_file)
+
+        yaml.contents.each do |content|
+          update_dir_in(content["directory"])
+        end
+      end
+    end
+
+    def update_dir(dir)
+      print_result do 
+        update_dir_in(dir)
+      end
+    end
+
+    def add(*content)
+      # 絶対パスに変換
+      content.map!{|v|File.expand_path(v)}
+
+      # YAML更新
+      yaml = yaml_load
+      yaml.add(*content)
+      yaml.save
+
+      # 部分アップデート
+      db_open(db_file)
+      content.each do |dir|
+        update_dir(dir)
+      end
+    end
+
+    def remove(*content)
+      # 絶対パスに変換
+      content.map!{|v|File.expand_path(v)}
+
+      # YAML更新
+      yaml = yaml_load
+      yaml.remove(*content)
+      yaml.save
+
+      # @todo 削除したコンテンツをインデックスから削除
+    end
+
+    def list
+      @out.puts yaml_load.list
+    end
+
+    def rebuild
+      db_delete(db_file)
+      db_create(db_file)
+      update
+    end
+
+    def dump
+      db_open(db_file)
+      
+      documents = Groonga::Context.default["documents"]
+      records = documents.select
+      records.each do |record|
+        @out.puts record.inspect
+        @out.puts "path : #{record.path}"
+        @out.puts "shortpath : #{record.shortpath}"
+        @out.puts "suffix : #{record.suffix}"
+        @out.puts "timestamp : #{record.timestamp.strftime('%Y/%m/%d %H:%M:%S')}"
+        @out.puts "content :", record.content ? record.content[0..64] : nil
+        @out.puts
+      end
+    end
+
+    private
+
+    def db_file
+      (Pathname.new(@db_dir) + DB_FILE_PATH).to_s
+    end
+
+    def yaml_file
+      GrendbYAML.yaml_file @db_dir
+    end
+
+    def yaml_load
+      GrendbYAML.load(@db_dir)
+    end
+
+    def update_dir_in(dir)
+      dir = File.expand_path(dir)
+
+      if (!FileTest.exist?(dir))
+        @out.puts "[WARNING]  : #{dir} (Not found, skip)"
+      elsif (FileTest.directory? dir)
+        db_add_dir(dir)
+      else
+        db_add_file(STDOUT, dir, File.basename(dir))
+      end
+    end
+
+    def time
+      @end_time - @start_time 
+    end
+
+    def print_result
+      yield
+      
+      @end_time = Time.now
+      
+      @out.puts
+      @out.puts "time       : #{Util::time_s(time)}"
+      @out.puts "files      : #{@file_count}"
+      @out.puts "add        : #{@add_count}"
+      @out.puts "update     : #{@update_count}"
+    end
+
+    def db_create(filename)
+      dbfile = Pathname(File.expand_path(filename))
+      dbdir = dbfile.dirname
+      dbdir.mkpath unless dbdir.exist?
+      
+      unless dbfile.exist?
+        Groonga::Database.create(:path => dbfile.to_s)
+        Groonga::Schema.define do |schema|
+          schema.create_table("documents") do |table|
+            table.string("path")
+            table.string("shortpath")
+            table.text("content")
+            table.time("timestamp")
+            table.text("suffix")
+          end
+
+          schema.create_table("terms",
+                              :type => :patricia_trie,
+                              :key_normalize => true,
+                              :default_tokenizer => "TokenBigram") do |table|
+            table.index("documents.path", :with_position => true)
+            table.index("documents.shortpath", :with_position => true)
+            table.index("documents.content", :with_position => true)
+            table.index("documents.suffix", :with_position => true)
+          end
+        end
+        @out.puts "create     : #{filename} created."
+      else
+        @out.puts "message    : #{filename} already exist."
+      end
+    end
+
+    def db_open(filename)
+      dbfile = Pathname(File.expand_path(filename))
+      
+      if dbfile.exist?
+        Groonga::Database.open(dbfile.to_s)
+        @out.puts  "open       : #{dbfile} open."
+      else
+        raise "error      : #{dbfile.to_s} not found!!"
+      end
+    end
+
+    def db_delete(filename)
+      raise "Illegal file name : #{filename}." unless filename =~ /\.db$/
+      Dir.glob("#{filename}*").each do |f|
+        @out.puts "delete     : #{f}"
+        FileUtils.rm_r(f)
+      end
+    end
+    private :db_delete
+      
+    def db_add_dir(dirname)
+      searchDirectory(STDOUT, dirname, File.basename(dirname), 0)
+    end
+    private :db_add_dir
+
+    def db_add_file(stdout, filename, shortpath)
+      # 格納するデータ
+      values = {
+        :path => filename,
+        :shortpath => shortpath,
+        :content => nil,
+        :timestamp => File.mtime(filename),
+        :suffix => File::extname(filename),
+      }
+      
+      # 検索するデータベース
+      documents = Groonga::Context.default["documents"]
+      
+      # 既に登録されているファイルならばそれを上書き、そうでなければ新規レコードを作成
+      _documents = documents.select do |record|
+        record["path"] == values[:path]
+      end
+      
+      isNewFile = false
+
+      if _documents.size.zero?
+        document = documents.add
+        isNewFile = true
+      else
+        document = _documents.to_a[0].key
+      end
+      
+      # タイムスタンプが新しければデータベースに格納
+      if (document[:timestamp] < values[:timestamp])
+        # 実際に使うタイミングでファイルの内容を読み込み
+        values[:content] = open(filename).read
+        
+        # データベースに格納
+        values.each do |key, value|
+          if (key == :path)
+            if (isNewFile)
+              @add_count += 1
+              @out.puts "add_file   : #{value}"
+            else
+              @update_count += 1
+              @out.puts "update     : #{value}"
+            end
+          end
+          document[key] = value
+        end
+      end
+
+    end
+
+    def searchDirectory(stdout, dir, shortdir, depth)
+      Dir.foreach(dir) do |name|
+        next if (name == '.' || name == '..')
+          
+        fpath = File.join(dir,name)
+        shortpath = File.join(shortdir,name)
+        
+        # 除外ディレクトリならばパス
+        next if ignoreDir?(fpath)
+
+        # 読み込み不可ならばパス
+        next unless FileTest.readable?(fpath)
+
+        # ファイルならば中身を探索、ディレクトリならば再帰
+        case File.ftype(fpath)
+        when "directory"
+          searchDirectory(stdout, fpath, shortpath, depth + 1)
+        when "file"
+          unless ignoreFile?(fpath)
+            db_add_file(stdout, fpath, shortpath)
+            @file_count += 1
+            @out.puts "file_count : #{@file_count}" if (@file_count % 100 == 0)
+          end
+        end          
+      end
+    end
+
+    def ignoreDir?(fpath)
+      FileTest.directory?(fpath) &&
+      GrenFileTest::ignoreDir?(fpath)
+    end
+    private :ignoreDir?
+
+    def ignoreFile?(fpath)
+      GrenFileTest::ignoreFile?(fpath) ||
+      GrenFileTest::binary?(fpath)
+    end
+    private :ignoreFile?
+
+  end
+end

data/test/file_test_utils.rb

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# @file
+# @brief ファイルテスト用ユーティリティ
+# @author ongaeshi
+# @date 2011/02/21
+#
+#
+# 以下のことを自動でやってくれます
+#
+# 1. tmpディレクトリの作成
+# 2. tmpディレクトリに移動
+# 3. テスト実行
+# 4. 元のディレクトリに戻る
+# 5. tmpディレクトリの削除
+#
+# また、以下の関数が使えます
+#
+# assert_diff_files(file1, file2)
+#
+# 二つのファイルが等しい場合はテスト成功
+# 失敗した場合は二つのファイルのdiffを表示します
+
+require 'pathname'
+require 'fileutils'
+
+module FileTestUtils
+  def setup
+    create_tmp_dir
+    FileUtils.cd(@tmp_dir.to_s)
+  end
+
+  def assert_diff_files(file1, file2)
+    unless (IO.read(file1) == IO.read(file2))
+      puts `diff -c #{file1} #{file2}`
+      assert_equal true, false
+    else
+      assert_equal true, true
+    end
+  end
+
+  def teardown
+    teardown_custom(true)
+  end
+
+  def teardown_custom(is_remove_dir)
+    FileUtils.cd(@prev_dir)
+    FileUtils.rm_rf(@tmp_dir.to_s) if (is_remove_dir)
+  end
+
+  private
+
+  def create_tmp_dir
+    @prev_dir = Dir.pwd
+    @tmp_dir = Pathname(File.dirname(__FILE__)) + "tmp"
+    FileUtils.rm_rf(@tmp_dir.to_s)
+    FileUtils.mkdir_p(@tmp_dir.to_s)
+  end
+end

data/test/runner.rb

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief
+# @author ongaeshi
+# @date   2011/03/11
+
+require 'test/unit'
+Test::Unit::AutoRunner.run(true, '.')
+
+

data/test/test_dbdir.rb

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief
+# @author ongaeshi
+# @date   2011/03/08
+
+require 'rubygems'
+require File.join(File.dirname(__FILE__), "test_helper")
+require File.join(File.dirname(__FILE__), "../lib/common/dbdir")
+require File.join(File.dirname(__FILE__), "file_test_utils")
+
+class TestDbDir< Test::Unit::TestCase
+  include CodeStock
+  include FileTestUtils
+
+  def test_db_default_dir
+    assert_equal File.expand_path('~/.codestock'), db_default_dir
+
+    ENV['CODESTOCK_DEFAULT_DIR'] = "~/DummyDir"
+    assert_equal File.expand_path("~/DummyDir"), db_default_dir
+
+    ENV['CODESTOCK_DEFAULT_DIR'] = nil
+    ENV['CODESTOCK_DEFAULT_DIRR'] = "~/DummyDir"
+    assert_equal File.expand_path('~/.codestock'), db_default_dir
+  end
+
+  def test_is_dbdir
+    assert_equal false, dbdir?
+
+    FileUtils.touch "grendb.yaml"
+    assert_equal true, dbdir?
+    
+    FileUtils.mkdir_p 'damadame'
+    FileUtils.touch "damadame/grendb.yaml"
+    assert_equal true, dbdir?('damadame')
+    assert_equal false, dbdir?('damadameyo')
+  end
+
+  def test_db_groonga_path
+    assert_equal 'db/grendb.db', db_groonga_path 
+    assert_equal '../db/grendb.db', db_groonga_path('..') 
+    assert_equal '/Users/MrRuby/db/grendb.db', db_groonga_path('/Users/MrRuby')
+  end
+
+  def test_db_expand_groonga_path
+    assert_equal File.expand_path('./db/grendb.db'), db_expand_groonga_path
+  end
+
+  def test_db_yaml_path
+    assert_equal 'grendb.yaml', db_yaml_path 
+    assert_equal '../grendb.yaml', db_yaml_path('..') 
+    assert_equal '/Users/MrRuby/grendb.yaml', db_yaml_path('/Users/MrRuby') 
+  end
+end
+
+
+
+

data/test/test_gren.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+
+class TestGren < Test::Unit::TestCase
+  def setup
+  end
+  
+  def test_truth
+    assert true
+  end
+end

data/test/test_gren_util.rb

@@ -0,0 +1,34 @@
+require File.join(File.dirname(__FILE__), 'test_helper')
+require File.join(File.dirname(__FILE__), '../lib/common/util')
+require File.join(File.dirname(__FILE__), '../lib/common/grensnip')
+
+class TestGrenSnip < Test::Unit::TestCase
+  def setup
+  end
+  
+  def test_snip
+    str = "abcdefghijkmlnopqrstuvwxyz"
+    assert_equal(str, GrenSnip::snip(str, nil))
+
+    str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|12345678901234567890123456789012345678901234567890123456"
+    assert_equal(str, GrenSnip::snip(str, nil))
+
+    str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
+    match_datas = []
+    match_datas << str.match(/123456789\|/)
+    assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
+
+    str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
+    match_datas = []
+    match_datas << str.match(/123456789\|/)
+    match_datas << str.match(/34567/)
+    assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>67890123456789012345678901234561", GrenSnip::snip(str, match_datas))
+
+    str = "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789|123456789012345678901234567890123456789012345678901234561"
+    match_datas = []
+    match_datas << str.match(/123456789\|/)
+    match_datas << str.match(/01234561/)
+    assert_equal("12345678901234567890123456789012<<snip>>90123456789012345678901234567890123456789|123456789012345678901234567890123<<snip>>8901234567890123456789012345678901234561", GrenSnip::snip(str, match_datas))
+
+  end
+end

data/test/test_grendbyaml.rb

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief
+# @author ongaeshi
+# @date   2011/02/20
+
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+require File.join(File.dirname(__FILE__), "../lib/mkgrendb/grendbyaml.rb")
+require 'fileutils'
+
+class TestGrendbYAML < Test::Unit::TestCase
+  include Mkgrendb
+
+  def setup
+    @prev_dir = Dir.pwd
+    @tmp_dir = Pathname(File.dirname(__FILE__)) + "tmp"
+    FileUtils.rm_rf(@tmp_dir.to_s)
+    FileUtils.mkdir_p(@tmp_dir.to_s)
+    FileUtils.cd(@tmp_dir.to_s)
+  end
+
+  def test_000
+    # create
+    yaml = GrendbYAML.create
+    assert_equal yaml.contents, []
+    assert_equal yaml.version, 0.1
+    assert_raise(GrendbYAML::YAMLAlreadyExist) { GrendbYAML.create }
+
+    # load
+    yaml = GrendbYAML.load
+    assert_equal yaml.contents, []
+    assert_equal yaml.version, 0.1
+
+    # load fail
+    FileUtils.mkdir 'loadtest'
+    FileUtils.cd 'loadtest' do
+      assert_raise(GrendbYAML::YAMLNotExist) { GrendbYAML.load }
+    end
+
+    # add
+    yaml.add('dir1')
+    yaml.add('dir2', 'dir3')
+    assert_equal ['dir1', 'dir2', 'dir3'], yaml.directorys
+
+    # remove
+    yaml.add('dir2', 'dir4', 'dir5')
+    yaml.remove('dir5')
+    yaml.remove('dir2', 'dir3')
+    assert_equal ['dir1', 'dir4'], yaml.directorys
+
+    # save
+    yaml.save
+    assert_equal <<EOF, open('grendb.yaml').read
+--- 
+version: 0.1
+contents: 
+- directory: dir1
+- directory: dir4
+EOF
+  end
+
+  def test_001
+    FileUtils.mkdir 'otherpath'
+    
+    # create
+    yaml = GrendbYAML.create('otherpath')
+    assert_equal yaml.contents, []
+    assert_equal yaml.version, 0.1
+    assert_raise(GrendbYAML::YAMLAlreadyExist) { GrendbYAML.create('otherpath') }
+
+    # load
+    yaml = GrendbYAML.load 'otherpath'
+    assert_equal yaml.contents, []
+    assert_equal yaml.version, 0.1
+
+    # load fail
+    FileUtils.mkdir 'loadtest'
+    FileUtils.cd 'loadtest' do
+      assert_raise(GrendbYAML::YAMLNotExist) { GrendbYAML.load }
+    end
+
+    # add
+    yaml.add('dir1')
+    yaml.add('dir2', 'dir3')
+    assert_equal ['dir1', 'dir2', 'dir3'], yaml.directorys
+
+    # remove
+    yaml.add('dir2', 'dir4', 'dir5')
+    yaml.remove('dir5')
+    yaml.remove('dir2', 'dir3')
+    assert_equal ['dir1', 'dir4'], yaml.directorys
+
+    # save
+    yaml.save
+    assert_equal <<EOF, open('otherpath/grendb.yaml').read
+--- 
+version: 0.1
+contents: 
+- directory: dir1
+- directory: dir4
+EOF
+  end
+  
+  def teardown
+    FileUtils.cd(@prev_dir)
+    FileUtils.rm_rf(@tmp_dir.to_s)
+  end
+end

data/test/test_grenweb_cli.rb

@@ -0,0 +1,10 @@
+require File.join(File.dirname(__FILE__), 'test_helper.rb')
+require File.join(File.dirname(__FILE__), '../lib/grenweb/cli.rb')
+require File.join(File.dirname(__FILE__), '../lib/grenweb/searcher.rb')
+require File.join(File.dirname(__FILE__), '../lib/grenweb/viewer.rb')
+
+class TestGrenwebCli < Test::Unit::TestCase
+  def test_print_default_output
+    # assert_match(/To update this executable/, @stdout)
+  end
+end

data/test/test_grenweb_database.rb

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+#
+# @file 
+# @brief
+# @author ongaeshi
+# @date   2011/02/20
+
+require 'rubygems'
+require 'groonga'
+require File.join(File.dirname(__FILE__), "test_helper")
+require File.join(File.dirname(__FILE__), "file_test_utils")
+require File.join(File.dirname(__FILE__), "../lib/grenweb/database")
+require File.join(File.dirname(__FILE__), "../lib/common/dbdir")
+require File.join(File.dirname(__FILE__), "../lib/mkgrendb/mkgrendb")
+require 'stringio'
+
+class TestMkgrendb < Test::Unit::TestCase
+  include Grenweb
+  include FileTestUtils
+  include CodeStock
+
+  def test_setup_and_open
+    io = StringIO.new
+    obj = ::Mkgrendb::Mkgrendb.new(io)
+    obj.init
+
+    Database.setup('.')
+    Database.instance
+  end
+
+  def teardown
+    teardown_custom(true)
+#    teardown_custom(false)
+  end
+end
+
+

data/test/test_grenweb_html_renderer.rb

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+require File.join(File.dirname(__FILE__), "test_helper.rb")
+require File.join(File.dirname(__FILE__), "../lib/grenweb/html_renderer.rb")
+
+class TestGrenwebHTMLRendeler < Test::Unit::TestCase
+  include Grenweb
+
+  def setup
+    @rendeler = HTMLRendeler.new('/')
+  end
+
+  def test_pagination_line
+    assert_equal("<span class='pagination-link'><a href='?page=1'>test</a></span>\n", @rendeler.pagination_link(1, "test"))
+  end
+  
+  def test_search_summary
+    assert_equal(@rendeler.search_summary(10, 500, 10..20, 0.00893),
+                 <<-EOS)
+  <div class='search-summary'>
+    <span class="keyword">10</span>の検索結果:
+    <span class="total-entries">500</span>件中
+    <span class="display-range">10 - 20</span>件（0.00893秒）
+  </div>
+EOS
+  end
+
+  def test_match_strong
+    assert_equal(@rendeler.match_strong("This is line.", [nil, nil]), "This is line.")
+  end
+
+  def test_search_box
+    assert_equal(@rendeler.search_box('test must'), <<-EOF)
+<form method="post" action="/::search">
+  <p>
+    <input name="query" type="text" size="60" value="test must" />
+    <input type="submit" value="検索" />
+  </p>
+</form>
+EOF
+  end
+end