gonzui 1.2

data/lib/gonzui.rb

@@ -0,0 +1,59 @@
+# -*- mode: ruby -*-
+#
+# gonzui - a source code search engine.
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+#require 'rubygems'
+
+$LOAD_PATH << "lib"
+$LOAD_PATH << "ext"
+
+module Gonzui
+  VERSION    = "1.2"
+  SYSCONFDIR = "."
+  PKGDATADIR = File.join(__FILE__, "..", "..", "data", "gonzui")
+  GONZUI_URI = "http://gonzui.sourceforge.net"
+  HTTP_PORT  = 46984
+  class GonzuiError < StandardError; end
+end
+
+require "gonzui/util"
+require "gonzui/gettext"
+
+begin # for Rake task
+require 'gonzui/autopack'
+require 'gonzui/delta'
+require "gonzui/texttokenizer"
+rescue LoadError
+end
+
+require "gonzui/dbm"
+require "gonzui/bdbdbm"
+require "gonzui/monitor"
+
+require "gonzui/content"
+require "gonzui/fetcher"
+require "gonzui/updater"
+require "gonzui/importer"
+require "gonzui/remover"
+require "gonzui/vcs"
+
+require "gonzui/apt"
+require "gonzui/config"
+require "gonzui/deindexer"
+require "gonzui/extractor"
+require "gonzui/indexer"
+require "gonzui/info"
+require "gonzui/license"
+require "gonzui/logger"
+require 'gonzui/searcher'
+require 'gonzui/searchquery'
+require 'gonzui/searchresult'
+
+

data/lib/gonzui/apt.rb

@@ -0,0 +1,193 @@
+#
+# apt.rb - an interface library for apt
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+module Gonzui
+  class AptGetError < GonzuiError; end
+
+  module AptGet
+    extend Util
+
+    AptGetRegistry = {}
+
+    module_function
+    def available?
+      begin
+        require_command("apt-get")
+        return true
+      rescue CommandNotFoundError
+        return false
+      end
+    end
+
+    def get_apt_type
+      require_command("apt-get")
+      apt_type = :unknown
+      IO.popen("apt-get --version").each {|line|
+        if m = /^\*Ver: Standard \.(.*)$/.match(line)
+          apt_type = m[1].intern
+        end
+      }
+      return apt_type
+    end
+
+    def new(config, package_name)
+      apt_type = get_apt_type
+      if klass = AptGetRegistry[apt_type]
+        return klass.new(config, package_name)
+      else
+        raise AptGetError.new("#{apt_type}: unsupported apt type")
+      end
+    end
+
+    def register(klass)
+      assert(!AptGetRegistry.include?(klass.apt_type))
+      AptGetRegistry[klass.apt_type] = klass
+    end
+  end
+
+  class AbstractAptGet
+    include Util
+    include TemporaryDirectoryUtil
+
+    def initialize(config, package_name)
+      @config = config
+      @package_name = package_name
+      @apt_options = ""
+      @cleaning_procs = []
+      set_temporary_directory(config.temporary_directory)
+    end
+
+    def run_apt_get
+	  cd = "cd"
+	  cd << " /d" if (/mswin|mingw|bccwin/ =~ RUBY_PLATFORM)
+      command_line = sprintf("#{cd} %s && apt-get -qq %s source %s >/dev/null 2>&1",
+                             shell_escape(self.temporary_directory),
+                             @apt_options,
+                             shell_escape(@package_name))
+      status = system(command_line)
+      raise AptGetError.new("#{@package_name}: unable to get sources") if
+        status == false
+    end
+
+    def extract_package
+      NotImplementedError.new("should be implemented in a sub class")
+    end
+
+    def add_cleaning_proc(proc)
+      @cleaning_procs.push(proc)
+    end
+
+    public
+    def extract
+      prepare_temporary_directory
+      run_apt_get
+      return extract_package
+    end
+
+    def clean
+      @cleaning_procs.each {|proc| proc.call }
+      clean_temporary_directory
+    end
+  end
+
+  class DebAptGet < AbstractAptGet
+    def self.apt_type
+      :deb
+    end
+
+    def find_archive(directory)
+      Dir.entries_without_dots(directory).map {|entry|
+        File.join(directory, entry)
+      }.find {|file_name|
+        Extractor.supported_file?(file_name)
+      }
+    end
+
+    def find_deb_source_directory
+      entries  = Dir.entries_without_dots(self.temporary_directory)
+      directory = entries.map {|entry|
+        File.join(self.temporary_directory, entry)
+      }.find {|path|
+        File.directory?(path)
+      }
+      raise AptGetError.new("#{@package_name}: source directory not found") if
+        directory.nil?
+      return directory
+    end
+
+    def remove_deb_files
+      Dir.entries(self.temporary_directory).each {|entry|
+        file_name = File.join(self.temporary_directory, entry)
+        File.unlink(file_name) if File.file?(file_name)
+      }
+    end
+
+    def contains_single_tarball?(directory)
+      entries = Dir.entries_without_dots(directory)
+      entries.delete("debian")
+      entries.delete("CVS") # some packages have it
+      entries.length == 1 and Extractor.supported_file?(entries.first)
+    end
+
+    def extract_package
+      remove_deb_files
+      source_directory = find_deb_source_directory
+      if contains_single_tarball?(source_directory)
+        archive_file_name = find_archive(source_directory)
+        extractor = Extractor.new(@config, archive_file_name)
+        add_cleaning_proc(lambda{ extractor.clean_temporary_directory })
+        return extractor.extract
+      else
+        return source_directory
+      end
+    end
+
+    AptGet.register(self)
+  end
+
+  class RPMAptGet < AbstractAptGet
+    def self.apt_type
+      :rpm
+    end
+
+    def initialize(config, package_name)
+      super(config, package_name)
+      @apt_options = "-d"
+    end
+
+    def srpm_file?(file_name)
+      /\.src\.rpm$/.match(file_name)
+    end
+
+    def find_srpm_file
+      entries = Dir.entries_without_dots(self.temporary_directory)
+      raise AptGetError.new("#{@package_name}: download failed") if
+        entries.length != 1
+
+      candidate = entries.first
+      raise AptGetError.new("#{@package_name}: SRPM not found") unless
+        srpm_file?(candidate)
+      return File.join(self.temporary_directory, candidate)
+    end
+
+    # FIXME: It cannot handle multiple tar balls in a single
+    # package like FC2's emacs including emacs, Mule-UCS, .
+    private
+    def extract_package
+      srpm_file_name = find_srpm_file
+      extractor = Extractor.new(@config, srpm_file_name)
+      add_cleaning_proc(lambda{ extractor.clean_temporary_directory })
+      return extractor.extract
+    end
+
+    AptGet.register(self)
+  end
+end

data/lib/gonzui/bdbdbm.rb

@@ -0,0 +1,118 @@
+#
+# bdbdbm.rb - bdb implementation of gonzui DB
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'bdb'
+
+module Gonzui
+  class BDBDBM < AbstractDBM
+    def initialize(config, read_only = false)
+      retried = false
+      begin
+        @db_mode  = if read_only then BDB::RDONLY else BDB::CREATE end
+        env_flags = BDB::CREATE | BDB::INIT_MPOOL | BDB::INIT_LOCK
+        options = {}
+        options["set_cachesize"] = config.db_cache_size
+        @db_env = BDB::Env.new(config.db_directory, env_flags, options)
+        super(config, read_only)
+      rescue BDB::Fatal => e
+        raise e if retried
+        retried = true
+        #
+        # "Lock table is out of available locker entries"
+        # error ocasionally occurs if a process using
+        # Gonzui::BDMDBM is killed by SIGKILL forcibly. In
+        # that case, we can remove Berkeley DB Environment
+        # to solve the error.
+        #
+        BDB::Env.remove(config.db_directory, BDB::FORCE)
+        retry
+      end
+    end
+
+    private
+    def do_open_db(name, key_type, value_type, dupsort)
+      options = {}
+      options["env"] = @db_env
+      if key_type == AutoPack::ID && !dupsort
+        options["set_array_base"] = 0
+        options["set_store_value"] = value_type.store if value_type.store
+        options["set_fetch_value"] = value_type.fetch if value_type.fetch
+        db = BDB::Recno.open(name.to_s, nil, @db_mode, 0644, options)
+      else
+        options["set_flags"] = BDB::DUPSORT if dupsort
+        options["set_store_key"] = key_type.store if key_type.store
+        options["set_fetch_key"] = key_type.fetch if key_type.fetch
+        options["set_store_value"] = value_type.store if value_type.store
+        options["set_fetch_value"] = value_type.fetch if value_type.fetch
+        db = BDB::Btree.open(name.to_s, nil, @db_mode, 0644, options)
+      end
+      db.extend(BDBExtension)
+      return db
+    end
+
+    public
+    def close
+      super
+      @db_env.close
+    end
+
+    def find_all_by_prefix(pattern)
+      results = []
+      self.word_wordid.each_by_prefix(pattern) {|word, word_id|
+        results.concat(collect_all_results(word_id))
+      }
+      return results
+    end
+  end
+
+  module BDBExtension
+	def duplicates(key , assoc = false)
+		super(key, assoc)
+	end
+
+    def each_by_prefix(prefix)
+      values = []
+      cursor = self.cursor
+      begin
+        if pair = cursor.set_range(prefix)
+          begin
+            k, v = pair
+            break unless k[0, prefix.length] == prefix
+            yield(k, v)
+          end while pair = cursor.next
+        end
+        return values
+      ensure
+        cursor.close
+      end
+    end
+
+    def delete_both(key, value)
+      cursor = self.cursor
+      begin
+        pair = cursor.get(BDB::GET_BOTH, key, value)
+        cursor.delete unless pair.nil?
+      ensure
+        cursor.close
+      end
+    end
+
+    def get_last_key
+      cursor = self.cursor
+      begin
+        k, v = cursor.last
+        return k
+      ensure
+        cursor.close
+      end
+    end
+  end
+end

data/lib/gonzui/cmdapp.rb

@@ -0,0 +1,14 @@
+#
+# cmdapp.rb - a library for command line applications
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'gonzui/cmdapp/app.rb'
+require 'gonzui/cmdapp/search.rb'
+

data/lib/gonzui/cmdapp/app.rb

@@ -0,0 +1,175 @@
+#
+# app.rb - command line application framework
+#
+# Copyright (C) 2004-2005 Satoru Takabayashi <satoru@namazu.org> 
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+
+require 'getoptlong'
+
+# gonzui applications use UTF-8 for the internal encoding.
+$KCODE = "u"
+
+module Gonzui
+  class CommandLineApplication
+    include Util
+    extend Util
+
+    def initialize
+      @config = Config.new
+
+      # ~/.gonzuirc has precedence over /etc/gonzuirc
+      file_names = []
+      file_names.push(File.join(ENV['HOME'], ".gonzuirc")) if ENV['HOME']
+      file_names.push(File.join(SYSCONFDIR, "gonzuirc"))
+      file_names.each {|file_name|
+        if File.exist?(file_name)
+          @config.load(file_name)
+          break
+        end
+      }
+    end
+
+    def parse_options_to_hash(option_table)
+      begin
+        options = {}
+        parser = GetoptLong.new
+        parser.set_options(*option_table)
+        parser.quiet = true
+        parser.each_option {|name, arg|
+          options[name.sub(/^--/, "")] = arg
+        }
+        return options
+      rescue => e
+        eprintf("%s", e.message)
+      end
+    end
+
+    def show_version
+      printf("%s %s\n", program_name, Gonzui::VERSION)
+      exit
+    end
+
+    def dump_config
+      @config.dump
+      exit
+    end
+
+    def be_quiet
+      devnull = if windows? then "NUL" else "/dev/null" end
+      STDOUT.reopen(devnull)
+    end
+
+    def ensure_db_directory_available
+      eprintf "#{@config.db_directory}: DB directory not found" unless
+        File.directory?(@config.db_directory)
+    end
+
+    def show_usage
+      do_show_usage
+      cache_size = @config.db_cache_size / 1024 ** 2
+      puts "      --gonzuirc=FILE            specify alternate run control file"
+      puts "      --dump-config              dump configuration"
+      puts "  -d, --db-dir=DIR               use DB directory DIR"
+      puts "      --db-cache=NUM             use NUM megabytes of DB cache [#{cache_size}]"
+      puts "      --list-formats             list all supported formats"
+      puts "      --help                     show this help message"
+      puts "  -q, --quiet                    suppress all normal output"
+      puts "  -v, --verbose                  output progress and statistics"
+      puts "      --version                  print version information and exit"
+      exit
+    end
+
+    def show_formats
+      max = LangScan::modules.sort_by {|m| m.name.length }.last.name.length
+      LangScan::modules.sort_by {|m| m.name }.each {|m|
+        printf("%-#{max}s %s\n", m.name, m.abbrev)
+      }
+      exit
+    end
+
+    def process_common_options(options)
+      show_version if options["version"]
+      show_usage if options["help"]
+      show_formats if options["list-formats"]
+      @config.quiet = true if options["quiet"]
+      @config.verbose = true if options["verbose"]
+      @config.load(File.expand_path(options["gonzuirc"])) if 
+        options["gonzuirc"]
+      @config.db_directory = File.expand_path(options["db-dir"]) if
+        options["db-dir"]
+      @config.db_cache_size = (options["db-cache"].to_f * 1024 ** 2).to_i if
+        options["db-cache"]
+
+      UTF8.set_preference(@config.encoding_preference)
+      set_verbosity(@config.verbose)
+      be_quiet if @config.quiet
+      dump_config if options["dump-config"]
+    end
+
+    def parse_options
+      option_table = [
+        ["--help",                GetoptLong::NO_ARGUMENT],
+        ["--version",             GetoptLong::NO_ARGUMENT],
+        ["--quiet",         "-q", GetoptLong::NO_ARGUMENT],
+        ["--verbose",       "-v", GetoptLong::NO_ARGUMENT],
+        ["--db-dir",        "-d", GetoptLong::REQUIRED_ARGUMENT],
+        ["--db-cache",            GetoptLong::REQUIRED_ARGUMENT],
+        ["--list-formats",        GetoptLong::NO_ARGUMENT],
+        ["--dump-config",         GetoptLong::NO_ARGUMENT],
+        ["--gonzuirc",            GetoptLong::REQUIRED_ARGUMENT]
+      ]
+      option_table.concat(do_get_option_table)
+      options = parse_options_to_hash(option_table)
+      process_common_options(options)
+      do_process_options(options)
+    end      
+
+    def init_logger
+      @logger = Logger.new(@config.gonzui_log_file)
+    end
+
+    def init_logger_with_stderr
+      @logger = Logger.new(@config.gonzui_log_file)
+      @logger.attach(STDERR)
+    end
+
+    def start
+      begin
+        do_start
+      rescue Interrupt, Errno::EPIPE
+      end
+    end
+
+
+    # Hook methods
+    def do_show_usage
+      raise NotImplementedError.new
+    end
+
+    def do_start
+      raise NotImplementedError.new
+    end
+
+    def do_process_options(options)
+      raise NotImplementedError.new
+    end
+
+    def do_get_option_table
+      raise NotImplementedError.new
+    end
+
+    def self.start
+      app = self.new
+      begin
+        app.start
+      rescue IncompatibleDBError => e
+        eprintf("%s", e.message)
+      end
+    end
+  end
+end