gonzui 1.2

data/Manifest.txt

@@ -0,0 +1,115 @@
+AUTHORS.txt
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+bin/gonzui-db
+bin/gonzui-import
+bin/gonzui-remove
+bin/gonzui-search
+bin/gonzui-server
+bin/gonzui-update
+data/gonzui/catalog/catalog.ja
+data/gonzui/doc/favicon.ico
+data/gonzui/doc/folder.png
+data/gonzui/doc/gonzui.css
+data/gonzui/doc/gonzui.js
+data/gonzui/doc/text.png
+data/gonzuirc.sample
+ext/autopack/autopack.c
+ext/autopack/extconf.rb
+ext/delta/delta.c
+ext/delta/extconf.rb
+ext/texttokenizer/extconf.rb
+ext/texttokenizer/texttokenizer.c
+ext/xmlformatter/extconf.rb
+ext/xmlformatter/xmlformatter.c
+lib/gonzui.rb
+lib/gonzui/apt.rb
+lib/gonzui/bdbdbm.rb
+lib/gonzui/cmdapp.rb
+lib/gonzui/cmdapp/app.rb
+lib/gonzui/cmdapp/search.rb
+lib/gonzui/config.rb
+lib/gonzui/content.rb
+lib/gonzui/dbm.rb
+lib/gonzui/deindexer.rb
+lib/gonzui/delta.rb
+lib/gonzui/extractor.rb
+lib/gonzui/fetcher.rb
+lib/gonzui/gettext.rb
+lib/gonzui/importer.rb
+lib/gonzui/indexer.rb
+lib/gonzui/info.rb
+lib/gonzui/license.rb
+lib/gonzui/logger.rb
+lib/gonzui/monitor.rb
+lib/gonzui/progressbar.rb
+lib/gonzui/remover.rb
+lib/gonzui/searcher.rb
+lib/gonzui/searchquery.rb
+lib/gonzui/searchresult.rb
+lib/gonzui/updater.rb
+lib/gonzui/util.rb
+lib/gonzui/vcs.rb
+lib/gonzui/webapp.rb
+lib/gonzui/webapp/advsearch.rb
+lib/gonzui/webapp/filehandler.rb
+lib/gonzui/webapp/jsfeed.rb
+lib/gonzui/webapp/markup.rb
+lib/gonzui/webapp/search.rb
+lib/gonzui/webapp/servlet.rb
+lib/gonzui/webapp/snippet.rb
+lib/gonzui/webapp/source.rb
+lib/gonzui/webapp/stat.rb
+lib/gonzui/webapp/top.rb
+lib/gonzui/webapp/uri.rb
+lib/gonzui/webapp/webrick.rb
+script/console
+script/destroy
+script/generate
+script/makemanifest.rb
+tasks/extconf.rake
+tasks/extconf/autopack.rake
+tasks/extconf/delta.rake
+tasks/extconf/texttokenizer.rake
+tasks/extconf/xmlformatter.rake
+test/_external_tools.rb
+test/_test-util.rb
+test/foo/Makefile.foo
+test/foo/bar.c
+test/foo/bar.h
+test/foo/foo.c
+test/foo/foo.spec
+test/test_apt.rb
+test/test_autopack_extn.rb
+test/test_bdbdbm.rb
+test/test_cmdapp-app.rb
+test/test_cmdapp-search.rb
+test/test_config.rb
+test/test_content.rb
+test/test_dbm.rb
+test/test_deindexer.rb
+test/test_delta.rb
+test/test_extractor.rb
+test/test_fetcher.rb
+test/test_gettext.rb
+test/test_gonzui.rb
+test/test_helper.rb
+test/test_importer.rb
+test/test_indexer.rb
+test/test_info.rb
+test/test_license.rb
+test/test_logger.rb
+test/test_monitor.rb
+test/test_searcher.rb
+test/test_searchquery.rb
+test/test_searchresult.rb
+test/test_texttokenizer.rb
+test/test_updater.rb
+test/test_util.rb
+test/test_vcs.rb
+test/test_webapp-markup.rb
+test/test_webapp-util.rb
+test/test_webapp-xmlformatter.rb

data/PostInstall.txt

@@ -0,0 +1,17 @@
+Welcome to gonzui!
+
+First, you need to install Berkley DB 4.7 and
+bdb for ruby (http://raa.ruby-lang.org/project/bdb/).
+If you use Ubuntu linux, only type following commands.
+ $ sudo apt-get install libdb-dev
+ $ sudo apt-get install libdb-ruby1.8
+
+First step:
+ $ mkdir gonzui
+ $ cd gonzui
+ $ gonzui-import path/to/project
+ $ gonzui-server
+ Access to http://localhost:46984/ and enjoy!
+
+More information:
+ See README.rdoc

data/README.rdoc

@@ -0,0 +1,149 @@
+= gonzui
+
+* http://gonzui.sourceforge.net/index.html.en
+* Gem version
+  * http://github.com/mallowlabs/gonzui
+
+== DESCRIPTION:
+
+gonzui is a source code search engine for accelerating open source software development. In the open source software development, programmers frequently refer to source codes written by others. Our goal is to help programmers develop programs effectively by creating a source code search engine that covers vast quantities of open source codes available on the Internet.
+
+== FEATURES/PROBLEMS:
+
+* Windows is not supported now.
+
+== SYNOPSIS:
+
+=== Basic Usage
+
+First, You need to import source codes into an index. For example,
+if you want to import the source codes of wget-1.9.1.tar.gz,
+run the following command.
+
+ % gonzui-import wget-1.9.1.tar.gz
+
+or if your system has apt-get and it's configured correctly,
+
+ % gonzui-import --apt wget
+
+Then, you can run gonzui-server that works as a web-based search engine.
+
+ % gonzui-server
+
+Finally, you can access the search engine with your browser.
+The URL is http://localhost:46984/.
+
+=== Import through version control systems
+
+Currently, gonzui supports import from CVS and Subversion, Git's repositories.
+For example, if you want to import source codes that can be obtained by the following command,
+
+ % cvs -d :pserver:anonymous@cvs.ruby-lang.org:/src co ruby
+
+you can just run gonzui-import as follows.
+
+ % gonzui-import --cvs :pserver:anonymous@cvs.ruby-lang.org:/src ruby
+
+In short, you can replace "cvs -d X co Y" with "gonzui-import --cvs X Y".
+
+For Subversion, if you want to import source codes that can be obtained by the following command,
+
+ % svn co http://svn.edgewall.com/repos/trac/trunk trac
+
+you can just run gonzui-import as follows.
+
+ % gonzui-import --svn http://svn.edgewall.com/repos/trac/trunk trac
+
+In short, you can replace "svn co X Y" with "gonzui-import --svn X Y".
+
+You can update the gonzui's index by gonzui-update command to reflect changes as described bellow.
+
+Currently, gonzui doesn't support revisions and tags of CVS and Subversion.
+
+=== Incremental Update
+
+gonzui-update is a tool for doing incremental update of the existing indices.
+For example, if you build an index by importing /somewhere/foo by the following command:
+
+ % gonzui-import /somewhere/foo
+
+and then, you want to update the existing index for reflecting
+recent changes (addition, deletion, and update of files) in /somewhere/foo,
+you can just run gonzui-update like:
+
+ % gonzui-update
+
+Since the index includes the location of /somewhere/foo,
+you don't have to give any parameters to gonzui-update command.
+If the existing index contains two or more packages,
+all packages are supposed to be updated.
+If /somewhere/foo is disappeared,
+gonzui-update prints an warning like "source disappeared".
+
+=== Other Usage
+
+Command-line-based search tool is also available mainly for debug purposes.
+To find locations of ``main'' function, run gonzui-search as follows.
+
+ % gonzui-search main
+
+You can inspect gonzui's index using gonzui-db command.
+To obtain the list of packages in the index, run gonzui-db as follows.
+
+ % gonzui-db --list
+
+To remove wget-1.9.1, run gonzui-remove as follows.
+
+ % gonzui-remove wget-1.9.1
+
+For further information on usage, try --help option of each command.
+
+== REQUIREMENTS:
+* Ruby 1.8.2+
+* bdb 4.2+ (Berkeley DB)
+ $ sudo apt-get install libdb-dev
+
+=== Ruby Libraries
+* langscan 1.2+
+ $ sudo gem install langscan
+* bdb for Ruby 0.5.2+
+ $ sudo apt-get install libdb-ruby1.8
+
+== Optional Software
+The following software is not required but useful.
+* Archive Handling
+  * tar (.tar.*)
+  * gunzip (.tar.gz)
+  * bunzip2 (.tar.bz2)
+  * uncompress (.tar.Z)
+  * unzip (.zip)
+  * lha (.lzh)
+  * rpm, rpmbuild (.src.rpm)
+* Source Code Fetcher
+  * apt-get (gonzui-import --apt)
+    * You need "dpkg-dev" package to fetch sources.
+* Version Control Systems
+  * cvs (gonzui-import --cvs)
+  * svn (gonzui-import --svn)
+  * git (gonzui-import --git)
+
+== INSTALL:
+
+ $ sudo gem install gonzui
+
+If you use Ubuntu, add following path to $PATH
+ /usr/lib/gems/1.8/bin
+
+== LICENSE:
+Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org>
+
+All rights reserved.
+
+This is free software with ABSOLUTELY NO WARRANTY.
+
+You can redistribute it and/or modify it under the terms of
+the GNU General Public License version 2.
+
+== AUTHORS:
+see 'AUTHORS.txt'
+

data/Rakefile

@@ -0,0 +1,28 @@
+require 'rubygems'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+require './lib/gonzui'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'gonzui' do
+  self.developer 'mallowlabs', 'mallowlabs@gmail.com'
+  self.post_install_message = File.read('PostInstall.txt')
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps         = [['langscan','>= 1.2']] #['bdb', '>= 0.5.2']
+
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]
+
+

data/bin/gonzui-db

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+#
+# gonzui-db - a tool to inspect a gonzui DB
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+# %LOADPATH%
+require 'getoptlong'
+require 'gonzui'
+require 'gonzui/cmdapp'
+
+include Gonzui
+include Gonzui::Util
+
+class GonzuiDB < CommandLineApplication
+  def command_list_files(package_names)
+    seen = {}
+    DBM.open(@config, true) {|dbm|
+      dbm.each_package_name {|package_name|
+        if package_names.empty? or package_names.find {|p| p == package_name }
+          seen[package_name] = true
+          package_id = dbm.get_package_id(package_name)
+          path_ids = dbm.get_path_ids(package_id)
+          path_ids.each {|path_id|
+            path = dbm.get_path(path_id)
+            printf("%s\n", path)
+          }
+        end
+      }
+    }
+    package_names.each {|package_name|
+      wprintf("%s: package not found", package_name) unless seen[package_name]
+    }
+  end
+
+  def command_list_packages(patterns)
+    DBM.open(@config, true) {|dbm|
+      dbm.each_package_name {|package_name|
+        if patterns.empty? or patterns.find {|p| /#{p}/.match(package_name) }
+          printf("%s\n", package_name)
+        end
+      }
+    }
+  end
+
+  def command_list_words(unused)
+    DBM.open(@config, true) {|dbm|
+      dbm.each_word {|word|
+        printf("%s\n", word)
+      }
+    }
+  end
+
+  def each_matching_db(db_names)
+    raise ArgmentError.new("block not given") unless block_given?
+    seen = {}
+    DBM.open(@config, true) {|dbm|
+      dbm.each_db_name {|db_name|
+        if db_names.empty? or db_names.find {|x| x == db_name }
+          seen[db_name] = true
+          db = dbm.send(db_name)
+          yield(db)
+        end
+      }
+    }
+    db_names.each {|db_name|
+      wprintf("%s: db not found", db_name) unless seen[db_name]
+    }
+  end
+
+  def command_list_keys(db_names)
+    each_matching_db(db_names) {|db|
+      db.each {|key, value|
+        printf("%s\n", key.inspect)
+      }
+    }
+  end
+
+  def command_retrieve(paths)
+    path = paths.first
+    DBM.open(@config, true) {|dbm|
+      path_id = dbm.get_path_id(path)
+      if path_id
+        print dbm.get_content(path_id)
+      else 
+        eprintf("%s: not found", path)
+      end
+    }
+  end
+
+  def command_dump(db_names)
+    each_matching_db(db_names) {|db|
+      db.each {|key, value|
+        printf("%s\t%s\n", key.inspect, value.inspect)
+      }
+    }
+  end
+
+  def command_verify(unsed)
+    DBM.open(@config, true) {|dbm|
+      begin
+        dbm.consistent?
+      rescue AssertionFailed => e
+        eprintf("DB is broken. You need to rebuild it.\n\n%s\n%s\n",
+                e.to_s,
+                e.backtrace.join("\n")
+                )
+      end
+    }
+  end
+
+  def do_show_usage
+    puts "Usage: #{program_name} [OPTION]..."
+    puts "  -l, --list [PATTERN]...        list packages"
+    puts "  -L, --list-files [PACKAGE]...  list files owend by package(s)"
+    puts "  -w, --list-words               list words"
+    puts "  -k, --list-keys [DBNAME]...    list keys in database(s) (for test)"
+    puts "      --dump [DBNAME]...         dump contents of database(s) (for test)"
+    puts "  -r, --retrieve PATH            retrieve the file of PATH"
+    puts "      --verify                   verify DB integrity"
+  end
+
+  def do_get_option_table
+    [
+      ['--list',         '-l', GetoptLong::NO_ARGUMENT],
+      ['--list-files',   '-L', GetoptLong::NO_ARGUMENT],
+      ['--list-words',   '-w', GetoptLong::NO_ARGUMENT],
+      ['--list-keys',    '-k', GetoptLong::NO_ARGUMENT],
+      ['--dump',               GetoptLong::NO_ARGUMENT],
+      ['--retrieve',     '-r', GetoptLong::NO_ARGUMENT],
+      ['--verify',             GetoptLong::NO_ARGUMENT],
+    ]
+  end
+
+  def do_process_options(options)
+    tasks = []
+    tasks.push(:command_list_files)    if options['list-files']
+    tasks.push(:command_list_packages) if options['list']
+    tasks.push(:command_list_words)  if options['list-words']
+    tasks.push(:command_list_keys)     if options['list-keys']
+    tasks.push(:command_retrieve)      if options['retrieve']
+    tasks.push(:command_dump)          if options['dump']
+    tasks.push(:command_verify)        if options['verify']
+    raise "cannot combine tasks" if tasks.length > 1
+    task = tasks.first
+
+    if task == nil or (task == :retrieve and ARGV.length != 1)
+      show_usage 
+    end
+    @task = task
+    @parameters = ARGV
+  end
+
+  def do_start
+    parse_options()
+    ensure_db_directory_available
+    send(@task, @parameters)
+  end
+end
+
+GonzuiDB.start