gonzui 1.2

data/test/test_texttokenizer.rb

@@ -0,0 +1,47 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require '_test-util'
+
+class TextTokenizerTest < Test::Unit::TestCase
+  include TestUtil
+
+  def _test_collect(text)
+    list = []
+    Gonzui::TextTokenizer.each_word(text) {|word, pos|
+      assert(word.is_a?(String))
+      assert(pos.is_a?(Integer))
+      list.push([word, pos])
+    }
+    return list
+  end
+
+  def _test_with_words(words, delim)
+    text = words.join(delim)
+    list = _test_collect(text)
+
+    pos = 0
+    words.each_with_index {|word, i|
+      assert_equal(word, list[i][0])
+      assert_equal(pos, list[i][1])
+      pos += word.length + delim.length
+    }
+  end
+
+  def _test_text(text, n)
+    list = _test_collect(text)
+    assert_equal(n, list.length)
+  end
+
+  def test_each
+    words = ["foo", "bar", "baz"]
+    _test_with_words(words, " ")
+    _test_with_words(words, "\n")
+    _test_with_words(words, ", ")
+    _test_with_words(words, ",\n")
+    _test_with_words(words, "!#%&\n\n\n@@")
+
+    _test_text("foo_bar", 1)
+    _test_text("あ", 1)  # single multi-byte character
+    _test_text("あい", 2) # two multi-byte characters
+  end
+end

data/test/test_updater.rb

@@ -0,0 +1,95 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require '_test-util'
+require 'fileutils'
+
+class UpdaterTest < Test::Unit::TestCase
+  include TestUtil
+
+  def setup
+	  @dbm = nil
+    @config   = Gonzui::Config.new
+  end
+  def teardown
+    unless @dbm.nil?
+      @dbm.close rescue nil
+    end
+    @dbm = nil
+    remove_db(@config)
+  end
+
+  def test_update
+    remove_db(@config)
+    tmp_dir = File.expand_path(
+      File.join(File.dirname(__FILE__), "tmp.update"))
+    #tmp_dir = File.expand_path("tmp.update")
+    FileUtils.rm_rf(tmp_dir)
+    FileUtils.mkdir(tmp_dir)
+    tmp_file1 = File.join(tmp_dir, "foo.txt")
+    tmp_file2 = File.join(tmp_dir, "bar.txt")
+    File.open(tmp_file1, "w") {|f| f.puts("foo") }
+
+    dbm = Gonzui::DBM.open(@config)
+    @dbm = dbm
+    url = URI.from_path(File.expand_path(tmp_file1))
+    content = Gonzui::Content.new(File.read(tmp_file1), File.mtime(tmp_file1))
+    source_url = URI.parse(sprintf("file://%s", tmp_dir))
+    indexer = Gonzui::Indexer.new(@config, dbm, source_url, tmp_file1, content)
+    indexer.index
+    dbm.flush_cache
+    assert_equal(1, dbm.get_ncontents)
+    assert(dbm.has_word?("foo"))
+    assert_equal(false, dbm.has_word?("bar"))
+    assert(dbm.consistent?)
+
+    File.open(tmp_file1, "w") {|f| f.puts("bar") }
+    updater = Gonzui::Updater.new(@config)
+    begin
+      updater.update
+      dbm.flush_cache
+      assert_equal(1, dbm.get_npackages)
+      assert_equal(1, dbm.get_ncontents)
+      assert_equal(1, dbm.get_nwords)
+      assert_equal(false, dbm.has_word?("foo"))
+      assert(dbm.has_word?("bar"))
+      assert(dbm.consistent?)
+    ensure
+      updater.finish
+    end
+
+    File.open(tmp_file2, "w") {|f| f.puts("baz") }
+    updater = Gonzui::Updater.new(@config)
+    begin
+      updater.update
+      dbm.flush_cache
+      assert_equal(1, dbm.get_npackages)
+      assert_equal(2, dbm.get_ncontents)
+      assert_equal(2, dbm.get_nwords)
+      assert(dbm.has_word?("bar"))
+      assert(dbm.has_word?("baz"))
+      assert(dbm.consistent?)
+    ensure
+      updater.finish
+    end
+
+    FileUtils.rm_rf(tmp_file1)
+    FileUtils.rm_rf(tmp_file2)
+    updater = Gonzui::Updater.new(@config)
+    begin
+      updater.update
+      dbm.flush_cache
+      assert_equal(0, dbm.get_npackages)
+      assert_equal(0, dbm.get_ncontents)
+      assert_equal(0, dbm.get_nwords)
+      assert_equal(false, dbm.has_word?("foo"))
+      assert_equal(false, dbm.has_word?("bar"))
+      assert_equal(false, dbm.has_word?("baz"))
+      assert(dbm.consistent?)
+    ensure
+      updater.finish
+    end
+
+    FileUtils.rm_rf(tmp_dir)
+  end
+end
+

data/test/test_util.rb

@@ -0,0 +1,149 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require 'ftools'
+require 'fileutils'
+require '_test-util'
+
+class UtilTest < Test::Unit::TestCase
+  include FileUtils
+  include Gonzui::Util
+
+  def test_string_methods
+    assert_equal("oo", "foo".prechop)
+    assert_equal("", "".prechop)
+    assert_equal(0...3, "foo\nbar\nbaz\n".line_range(0))
+    assert_equal(0...3, "foo\nbar\nbaz\n".line_range(2))
+    assert_equal(4...7, "foo\nbar\nbaz\n".line_range(4))
+
+    "foo\nbar\nbaz\n".each_line_range(4, 1) {|lineno_offset, range|
+      case lineno_offset
+      when -1
+        assert_equal(0...3, range)
+      when 0
+        assert_equal(4...7, range)
+      when 1
+        assert_equal(8...11, range)
+      else
+        assert(false)
+      end
+    }
+  end
+
+  def test_dir_methods
+    rm_rf("t")
+    File.mkpath("t")
+    assert_equal([], Dir.entries_without_dots("t"))
+
+    touch(["t/a", "t/b"])
+    assert_equal(["a", "b"], Dir.entries_without_dots("t"))
+
+    File.mkpath("t/t")
+    assert_equal(["a", "b", "t"], Dir.entries_without_dots("t"))
+
+    touch(["t/t/c"])
+    assert_equal(["t/a", "t/b", "t/t/c", ], Dir.all_files("t"))
+    rm_rf("t")
+  end
+
+  def test_file_methods
+    assert_equal("bar", File.relative_path("/foo/bar", "/foo"))
+    assert_equal("foo", File.relative_path("/foo", "/"))
+    assert_equal("", File.relative_path("/foo", "/foo"))
+    assert(File.any_exist?("/"))
+  end
+
+  def test_command_operations
+    unless windows?
+      assert(command_exist?("sh")) 
+      assert(require_command("sh"))
+    end
+  end
+
+  def test_shell_escape
+    assert_equal('"foo"', shell_escape('foo'))
+    if (/mswin|mingw|bccwin/ =~ RUBY_PLATFORM)
+      assert_equal('"c:\\temp"', shell_escape('c:/temp'))
+      assert_equal('"file:///c:/temp"', shell_escape('file://c:/temp'))
+      assert_equal('"file:///c:/temp"', shell_escape('file:///c:/temp'))
+    else
+      assert_equal('"\\$foo"', shell_escape('$foo'))
+      assert_equal('"\\`foo\\`"', shell_escape('`foo`'))
+      assert_equal('"foo\\\\bar"', shell_escape('foo\\bar'))
+    end
+  end
+
+  def test_commify
+    assert_equal("1",         commify(1))
+    assert_equal("12",        commify(12))
+    assert_equal("123",       commify(123))
+    assert_equal("1,234",     commify(1234))
+    assert_equal("12,345",    commify(12345))
+    assert_equal("123,456",   commify(123456))
+    assert_equal("1,234,567", commify(1234567))
+  end
+
+  def test_temporary_directory_util
+    foo = Object.new
+    foo.extend(Gonzui::TemporaryDirectoryUtil)
+    foo.set_temporary_directory(".")
+    foo.prepare_temporary_directory
+    begin
+      foo.prepare_temporary_directory
+      assert(false)
+    rescue
+      assert(true)
+    end
+    assert(File.directory?(foo.temporary_directory))
+    foo.clean_temporary_directory
+    assert_equal(false, File.exist?(foo.temporary_directory))
+  end
+
+  def test_utf8
+    str, name = Gonzui::UTF8.to_utf8("foo")
+    assert_equal("foo", str)
+    assert_equal("us-ascii", name)
+
+
+    utf8 = "日本語です"
+    ["shift_jis", "euc-jp", "iso-2022-jp"].each {|name|
+      str = Iconv.conv(name, "utf-8", utf8)
+      s, n = Gonzui::UTF8.to_utf8(str)
+      assert_equal(name, n)
+      assert_equal(utf8, s)
+    }
+  end
+
+  def text_windows_path
+  	# drive leter
+  	path = "C:\\WINDOWS\\"
+  	r = from_path(path)
+    assert_equal("C:", r.host)
+    assert_equal("\\WINDOWS\\", r.path)
+    assert_equal(path, r.to_s)
+
+  	path = "C:/WINDOWS/"
+  	r = from_path(path)
+    assert_equal("C:", r.host)
+    assert_equal("/WINDOWS/", r.path)
+
+  	path = "C:/WINDOWS"
+  	r = from_path(path)
+    assert_equal("C:", r.host)
+    assert_equal("/WINDOWS", r.path)
+
+	# space
+  	path = "C:\\Documents and Settings\\"
+  	r = from_path(path)
+    assert_equal("C:", r.host)
+    assert_equal("\\Documents and Settings\\", r.path)
+    assert_equal(path, r.to_s)
+
+	# extend
+  	path = "\\\\?\\C:\\WINDOWS\\"
+  	r = from_path(path)
+    assert_equal("\\\\", r.host)
+    assert_equal("?\\C:\\WINDOWS\\", r.path)
+    assert_equal(path, r.to_s)
+  end
+end
+

data/test/test_vcs.rb

@@ -0,0 +1,61 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require '_external_tools'
+require 'ftools'
+require '_test-util'
+
+class VCSTest < Test::Unit::TestCase
+  include TestUtil
+  include Gonzui::Util
+
+  def teardown
+    remove_db(Gonzui::Config.new)
+  end
+
+  def prepare
+    make_dist_tree
+    config = Gonzui::Config.new
+    config.quiet = true
+    remove_db(config)
+    return config
+  end
+
+  def test_cvs
+    config = prepare
+    make_cvs
+    cvs = Gonzui::CVS.new(config, cvsroot, "foo")
+    cvs.extract
+    cached_foo = File.join(config.cache_directory, "foo")
+    entries = Dir.entries_without_dots(cached_foo)
+    FOO_FILES.each {|base_name|
+      assert(entries.include?(base_name))
+    }
+    remove_cvs
+  end if (CVS_)
+
+  def test_svn
+    config = prepare
+    svnroot_uri = make_svn
+    svn = Gonzui::Subversion.new(config, svnroot_uri, "foo")
+    svn.extract
+    cached_foo = File.join(config.cache_directory, "foo")
+    entries = Dir.entries_without_dots(cached_foo)
+    FOO_FILES.each {|base_name|
+      assert(entries.include?(base_name))
+    }
+    remove_svn
+  end if (SVN_)
+
+  def test_git
+    config = prepare
+    gitroot = make_git
+    git = Gonzui::Git.new(config, gitroot, "foo")
+    git.extract
+    cached_foo = File.join(config.cache_directory, File.basename(gitroot))
+    entries = Dir.entries_without_dots(cached_foo)
+    FOO_FILES.each {|base_name|
+      assert(entries.include?(base_name))
+    }
+    remove_git
+  end if (GIT_)
+end if (CVS_ || SVN_ || GIT_)

data/test/test_webapp-markup.rb

@@ -0,0 +1,42 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require '_external_tools.rb'
+require 'gonzui/webapp'
+require '_test-util'
+
+class TextBeautifierTest < Test::Unit::TestCase
+  include TestUtil
+
+  def setup
+	@dbm = nil
+  end
+  def teardown
+    unless @dbm.nil?
+      @dbm.close rescue nil
+    end
+    @dbm = nil
+  end
+
+  def init_dbm(config)
+    remove_db(config)
+    make_archives
+    add_package(config)
+    dbm = Gonzui::DBM.open(config)
+    @dbm = dbm
+    return dbm
+  end
+
+  def test_beautify
+    config = Gonzui::Config.new
+    dbm = init_dbm(config)
+    path_id = dbm.get_path_id("foo-0.1/foo.c")
+    assert(path_id.is_a?(Fixnum))
+    content = dbm.get_content(path_id)
+    digest  = dbm.get_digest(path_id)
+    beautifier = Gonzui::TextBeautifier.new(content, digest, [], "search?q=")
+    html = beautifier.beautify
+    assert(html.is_a?(Array))
+    assert(html.flatten.include?(:span))
+    remove_db(config)
+  end
+end if (ARC_)

data/test/test_webapp-util.rb

@@ -0,0 +1,19 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require 'gonzui/webapp/xmlformatter'
+require '_test-util'
+
+class UtilTest < Test::Unit::TestCase
+  def test_format_xml
+    html = [:html]
+    head = [:head, [:title, "foo"]]
+    body = [:body, [:h1, [:a, {:href => 'foo<&">.html'}, 'foo<&">']]]
+    body.push([:p, "hello"])
+    html.push(head)
+    html.push(body)
+    formatter = Gonzui::XMLFormatter.new
+    result = formatter.format(html)
+    assert_equal("<html\n><head\n><title\n>foo</title\n></head\n><body\n><h1\n><a href=\"foo&lt;&amp;&quot;&gt;.html\"\n>foo&lt;&amp;&quot;&gt;</a\n></h1\n><p\n>hello</p\n></body\n></html\n>",
+                 result)
+  end
+end

data/test/test_webapp-xmlformatter.rb

@@ -0,0 +1,19 @@
+#! /usr/bin/env ruby
+require File.dirname(__FILE__) + '/test_helper.rb'
+require 'gonzui/webapp/xmlformatter'
+require '_test-util'
+
+class XMLFormatterTest < Test::Unit::TestCase
+  def test_format_xml
+    html = [:html]
+    head = [:head, [:title, "foo"]]
+    body = [:body, [:h1, [:a, {:href => 'foo<&">.html'}, 'foo<&">']]]
+    body.push([:p, "hello"])
+    html.push(head)
+    html.push(body)
+    formatter = Gonzui::XMLFormatter.new
+    result = formatter.format(html)
+    assert_equal("<html\n><head\n><title\n>foo</title\n></head\n><body\n><h1\n><a href=\"foo&lt;&amp;&quot;&gt;.html\"\n>foo&lt;&amp;&quot;&gt;</a\n></h1\n><p\n>hello</p\n></body\n></html\n>",
+                 result)
+  end
+end

metadata

@@ -0,0 +1,291 @@
+--- !ruby/object:Gem::Specification 
+name: gonzui
+version: !ruby/object:Gem::Version 
+  hash: 11
+  prerelease: false
+  segments: 
+  - 1
+  - 2
+  version: "1.2"
+platform: ruby
+authors: 
+- mallowlabs
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-15 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: langscan
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 1
+        - 2
+        version: "1.2"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rubyforge
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 21
+        segments: 
+        - 2
+        - 6
+        - 1
+        version: 2.6.1
+  type: :development
+  version_requirements: *id003
+description: gonzui is a source code search engine for accelerating open source software development. In the open source software development, programmers frequently refer to source codes written by others. Our goal is to help programmers develop programs effectively by creating a source code search engine that covers vast quantities of open source codes available on the Internet.
+email: 
+- mallowlabs@gmail.com
+executables: 
+- gonzui-db
+- gonzui-import
+- gonzui-remove
+- gonzui-search
+- gonzui-server
+- gonzui-update
+extensions: 
+- ext/autopack/extconf.rb
+- ext/delta/extconf.rb
+- ext/texttokenizer/extconf.rb
+- ext/xmlformatter/extconf.rb
+extra_rdoc_files: 
+- AUTHORS.txt
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+files: 
+- AUTHORS.txt
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+- Rakefile
+- bin/gonzui-db
+- bin/gonzui-import
+- bin/gonzui-remove
+- bin/gonzui-search
+- bin/gonzui-server
+- bin/gonzui-update
+- data/gonzui/catalog/catalog.ja
+- data/gonzui/doc/favicon.ico
+- data/gonzui/doc/folder.png
+- data/gonzui/doc/gonzui.css
+- data/gonzui/doc/gonzui.js
+- data/gonzui/doc/text.png
+- data/gonzuirc.sample
+- ext/autopack/autopack.c
+- ext/autopack/extconf.rb
+- ext/delta/delta.c
+- ext/delta/extconf.rb
+- ext/texttokenizer/extconf.rb
+- ext/texttokenizer/texttokenizer.c
+- ext/xmlformatter/extconf.rb
+- ext/xmlformatter/xmlformatter.c
+- lib/gonzui.rb
+- lib/gonzui/apt.rb
+- lib/gonzui/bdbdbm.rb
+- lib/gonzui/cmdapp.rb
+- lib/gonzui/cmdapp/app.rb
+- lib/gonzui/cmdapp/search.rb
+- lib/gonzui/config.rb
+- lib/gonzui/content.rb
+- lib/gonzui/dbm.rb
+- lib/gonzui/deindexer.rb
+- lib/gonzui/delta.rb
+- lib/gonzui/extractor.rb
+- lib/gonzui/fetcher.rb
+- lib/gonzui/gettext.rb
+- lib/gonzui/importer.rb
+- lib/gonzui/indexer.rb
+- lib/gonzui/info.rb
+- lib/gonzui/license.rb
+- lib/gonzui/logger.rb
+- lib/gonzui/monitor.rb
+- lib/gonzui/progressbar.rb
+- lib/gonzui/remover.rb
+- lib/gonzui/searcher.rb
+- lib/gonzui/searchquery.rb
+- lib/gonzui/searchresult.rb
+- lib/gonzui/updater.rb
+- lib/gonzui/util.rb
+- lib/gonzui/vcs.rb
+- lib/gonzui/webapp.rb
+- lib/gonzui/webapp/advsearch.rb
+- lib/gonzui/webapp/filehandler.rb
+- lib/gonzui/webapp/jsfeed.rb
+- lib/gonzui/webapp/markup.rb
+- lib/gonzui/webapp/search.rb
+- lib/gonzui/webapp/servlet.rb
+- lib/gonzui/webapp/snippet.rb
+- lib/gonzui/webapp/source.rb
+- lib/gonzui/webapp/stat.rb
+- lib/gonzui/webapp/top.rb
+- lib/gonzui/webapp/uri.rb
+- lib/gonzui/webapp/webrick.rb
+- script/console
+- script/destroy
+- script/generate
+- script/makemanifest.rb
+- tasks/extconf.rake
+- tasks/extconf/autopack.rake
+- tasks/extconf/delta.rake
+- tasks/extconf/texttokenizer.rake
+- tasks/extconf/xmlformatter.rake
+- test/_external_tools.rb
+- test/_test-util.rb
+- test/foo/Makefile.foo
+- test/foo/bar.c
+- test/foo/bar.h
+- test/foo/foo.c
+- test/foo/foo.spec
+- test/test_apt.rb
+- test/test_autopack_extn.rb
+- test/test_bdbdbm.rb
+- test/test_cmdapp-app.rb
+- test/test_cmdapp-search.rb
+- test/test_config.rb
+- test/test_content.rb
+- test/test_dbm.rb
+- test/test_deindexer.rb
+- test/test_delta.rb
+- test/test_extractor.rb
+- test/test_fetcher.rb
+- test/test_gettext.rb
+- test/test_gonzui.rb
+- test/test_helper.rb
+- test/test_importer.rb
+- test/test_indexer.rb
+- test/test_info.rb
+- test/test_license.rb
+- test/test_logger.rb
+- test/test_monitor.rb
+- test/test_searcher.rb
+- test/test_searchquery.rb
+- test/test_searchresult.rb
+- test/test_texttokenizer.rb
+- test/test_updater.rb
+- test/test_util.rb
+- test/test_vcs.rb
+- test/test_webapp-markup.rb
+- test/test_webapp-util.rb
+- test/test_webapp-xmlformatter.rb
+has_rdoc: true
+homepage: http://gonzui.sourceforge.net/index.html.en
+licenses: []
+
+post_install_message: |
+  Welcome to gonzui!
+  
+  First, you need to install Berkley DB 4.7 and
+  bdb for ruby (http://raa.ruby-lang.org/project/bdb/).
+  If you use Ubuntu linux, only type following commands.
+   $ sudo apt-get install libdb-dev
+   $ sudo apt-get install libdb-ruby1.8
+  
+  First step:
+   $ mkdir gonzui
+   $ cd gonzui
+   $ gonzui-import path/to/project
+   $ gonzui-server
+   Access to http://localhost:46984/ and enjoy!
+  
+  More information:
+   See README.rdoc
+
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+- ext/autopack
+- ext/delta
+- ext/texttokenizer
+- ext/xmlformatter
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: gonzui
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: gonzui is a source code search engine for accelerating open source software development
+test_files: 
+- test/test_apt.rb
+- test/test_autopack_extn.rb
+- test/test_bdbdbm.rb
+- test/test_cmdapp-app.rb
+- test/test_cmdapp-search.rb
+- test/test_config.rb
+- test/test_content.rb
+- test/test_dbm.rb
+- test/test_deindexer.rb
+- test/test_delta.rb
+- test/test_extractor.rb
+- test/test_fetcher.rb
+- test/test_gettext.rb
+- test/test_gonzui.rb
+- test/test_helper.rb
+- test/test_importer.rb
+- test/test_indexer.rb
+- test/test_info.rb
+- test/test_license.rb
+- test/test_logger.rb
+- test/test_monitor.rb
+- test/test_searcher.rb
+- test/test_searchquery.rb
+- test/test_searchresult.rb
+- test/test_texttokenizer.rb
+- test/test_updater.rb
+- test/test_util.rb
+- test/test_vcs.rb
+- test/test_webapp-markup.rb
+- test/test_webapp-util.rb
+- test/test_webapp-xmlformatter.rb