bio 0.7.0

data/lib/bio/io/flatfile/bdb.rb

@@ -0,0 +1,266 @@
+# 
+# bio/io/flatfile/bdb.rb - OBDA flatfile index by Berkley DB 
+# 
+#   Copyright (C) 2002 GOTO Naohisa <ngoto@gen-info.osaka-u.ac.jp> 
+# 
+#  This library is free software; you can redistribute it and/or 
+#  modify it under the terms of the GNU Lesser General Public 
+#  License as published by the Free Software Foundation; either 
+#  version 2 of the License, or (at your option) any later version. 
+# 
+#  This library is distributed in the hope that it will be useful, 
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of 
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
+#  Lesser General Public License for more details. 
+# 
+#  You should have received a copy of the GNU Lesser General Public 
+#  License along with this library; if not, write to the Free Software 
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA 
+# 
+#  $Id: bdb.rb,v 1.8 2005/09/26 13:00:08 k Exp $ 
+# 
+ 
+begin 
+  require 'bdb' 
+rescue LoadError,NotImplementedError
+end 
+
+require 'bio/io/flatfile/index'
+require 'bio/io/flatfile/indexer'
+
+module Bio
+  class FlatFileIndex
+
+    module BDBdefault
+      def permission
+        (0666 & (0777 ^ File.umask))
+      end
+      module_function :permission
+
+      def flag_read
+        BDB::RDONLY
+      end
+      module_function :flag_read
+
+      def flag_write
+        (BDB::CREATE | BDB::TRUNCATE)
+      end
+      module_function :flag_write
+
+      def flag_append
+        'r+'
+      end
+      module_function :flag_append
+    end #module BDBdefault
+
+    class BDBwrapper
+      def initialize(name, filename, *arg)
+        @dbname = name
+        @file = nil
+        @filename = filename
+        #self.open(*arg)
+      end
+
+      def filename
+        File.join(@dbname, @filename)
+      end
+
+      def open(flag = BDBdefault.flag_read,
+               permission = BDBdefault.permission)
+        unless @file then
+          DEBUG.print "BDBwrapper: open #{filename}\n"
+          @file = BDB::Btree.open(filename, nil, flag, permission)
+        end
+        true
+      end
+
+      def close
+        if @file
+          DEBUG.print "BDBwrapper: close #{filename}\n"
+          @file.close
+          @file = nil
+        end
+        nil
+      end
+
+      def [](arg)
+        #self.open
+        if @file then
+          @file[arg]
+        else
+          nil
+        end
+      end
+
+      def []=(key, val)
+        #self.open
+        @file[key.to_s] = val.to_s
+      end
+
+      def writeback_array(prefix, array, *arg)
+        self.close
+        self.open(*arg)
+        array.each_with_index do |val, key|
+          @file["#{prefix}#{key}"] = val.to_s
+        end
+      end
+
+      def keys
+        if @file then
+          @file.keys
+        else
+          []
+        end
+      end
+    end #class BDBwrapper
+
+    module BDB_1
+      class BDBMappingFile
+        def self.open(*arg)
+          self.new(*arg)
+        end
+
+        def initialize(filename, flag = BDBdefault.flag_read,
+                       permission = BDBdefault.permission)
+          @filename = filename
+          @flag = flag
+          @permission = permission
+          #@bdb = BDB::Btree.open(@filename, nil, @flag, @permission)
+        end
+        attr_reader :filename
+        attr_accessor :flag, :permission
+
+        def open
+          unless @bdb then
+            DEBUG.print "BDBMappingFile: open #{@filename}\n"
+            @bdb = BDB::Btree.open(@filename, nil, @flag, @permission)
+            true
+          else
+            nil
+          end
+        end
+
+        def close
+          if @bdb then
+            DEBUG.print "BDBMappingFile: close #{@filename}\n"
+            @bdb.close
+            @bdb = nil
+          end
+          nil
+        end
+
+        def records
+          @bdb.size
+        end
+        alias size records
+
+        # methods for writing
+        def add(key, val)
+          open
+          val = val.to_a.join("\t")
+          s = @bdb[key]
+          if s then
+            s << "\t"
+            s << val
+            val = s
+          end
+          @bdb[key] = val
+          #DEBUG.print "add: key=#{key.inspect}, val=#{val.inspect}\n"
+          val
+        end
+
+        def add_exclusive(key, val)
+          open
+          val = val.to_a.join("\t")
+          s = @bdb[key]
+          if s then
+            raise RuntimeError, "keys must be unique, but key #{key.inspect} already exists"
+          end
+          @bdb[key] = val
+          #DEBUG.print "add_exclusive: key=#{key.inspect}, val=#{val.inspect}\n"
+          val
+        end
+
+        def add_overwrite(key, val)
+          open
+          val = val.to_a.join("\t")
+          s = @bdb[key]
+          if s then
+            DEBUG.print "Warining: overwrote unique id #{key.inspect}\n"
+          end
+          @bdb[key] = val
+          #DEBUG.print "add_overwrite: key=#{key.inspect}, val=#{val.inspect}\n"
+          val
+        end
+
+        def add_nr(key, val)
+          open
+          s = @bdb[key]
+          if s then
+            a = s.split("\t")
+          else
+            a = []
+          end
+          a.concat val.to_a
+          a.sort!
+          a.uniq!
+          str = a.join("\t")
+          @bdb[key] = str
+          #DEBUG.print "add_nr: key=#{key.inspect}, val=#{str.inspect}\n"
+          str
+        end
+            
+        # methods for searching
+        def search(key)
+          open
+          s = @bdb[key]
+          if s then
+            a = s.split("\t")
+            a
+          else
+            []
+          end
+        end
+      end #class BDBMappingFile
+
+      class PrimaryNameSpace < Template::NameSpace
+        def mapping(filename)
+          BDBMappingFile.new(filename)
+        end
+        def filename
+          File.join(dbname, "key_#{name}")
+        end
+        def search(key)
+          r = super(key)
+          unless r.empty? then
+            [ r ]
+          else
+            r
+          end
+        end
+      end #class PrimaryNameSpace
+
+      class SecondaryNameSpace < Template::NameSpace
+        def mapping(filename)
+          BDBMappingFile.new(filename)
+        end
+        def filename
+          File.join(dbname, "id_#{name}")
+        end #class SecondaryNameSpaces
+      
+        def search(key)
+          r = super(key)
+          file.close
+          r
+        end
+      end #class SecondaryNameSpace
+    end #module BDB_1
+
+  end #class FlatFileIndex
+end #module Bio
+
+=begin
+
+  * Classes/modules in this file are internal use only.
+
+=end

data/lib/bio/io/flatfile/index.rb

@@ -0,0 +1,1308 @@
+# 
+# bio/io/flatfile/index.rb - OBDA flatfile index 
+# 
+#   Copyright (C) 2002 GOTO Naohisa <ngoto@gen-info.osaka-u.ac.jp> 
+# 
+#  This library is free software; you can redistribute it and/or 
+#  modify it under the terms of the GNU Lesser General Public 
+#  License as published by the Free Software Foundation; either 
+#  version 2 of the License, or (at your option) any later version. 
+# 
+#  This library is distributed in the hope that it will be useful, 
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of 
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU 
+#  Lesser General Public License for more details. 
+# 
+#  You should have received a copy of the GNU Lesser General Public 
+#  License along with this library; if not, write to the Free Software 
+#  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA 
+# 
+#  $Id: index.rb,v 1.15 2005/11/28 05:08:26 k Exp $ 
+# 
+
+require 'bio/io/flatfile/indexer'
+
+module Bio
+  class FlatFileIndex
+
+    autoload :Indexer,    'bio/io/flatfile/indexer'
+    autoload :BDBdefault, 'bio/io/flatfile/bdb'
+    autoload :BDBwrapper, 'bio/io/flatfile/bdb'
+    autoload :BDB_1,      'bio/io/flatfile/bdb'
+
+    MAGIC_FLAT = 'flat/1'
+    MAGIC_BDB  = 'BerkeleyDB/1'
+
+    #########################################################
+    def self.open(name)
+      if block_given? then
+        begin
+          i = self.new(name)
+          r = yield i
+        ensure
+          if i then
+            begin
+              i.close
+            rescue IOError
+            end
+          end
+        end
+      else
+        r = self.new(name)
+      end
+      r
+    end
+
+    def initialize(name)
+      @db = DataBank.open(name)
+    end
+
+    # common interface defined in registry.rb
+    def get_by_id(key)
+      search(key).to_s
+    end
+
+    # original methods
+    def close
+      check_closed?
+      @db.close
+      @db = nil
+    end
+
+    def closed?
+      if @db then
+        false
+      else
+        true
+      end
+    end
+
+    def default_namespaces=(names)
+      if names then
+        @names = []
+        names.each { |x| @names.push(x.dup) }
+      else
+        @names = nil
+      end
+    end
+
+    def default_namespaces
+      @names
+    end
+
+    def search(key)
+      check_closed?
+      if @names then
+        @db.search_namespaces(key, *@names)
+      else
+        @db.search_all(key)
+      end
+    end
+
+    def search_namespaces(key, *names)
+      check_closed?
+      @db.search_namespaces(key, *names)
+    end
+
+    def search_primary(key)
+      check_closed?
+      @db.search_primary(key)
+    end
+
+    def include?(key)
+      check_closed?
+      if @names then
+        r = @db.search_namespaces_get_unique_id(key, *@names)
+      else
+        r = @db.search_all_get_unique_id(key)
+      end
+      if r.empty? then
+        nil
+      else
+        r
+      end
+    end
+
+    def include_in_namespaces?(key, *names)
+      check_closed?
+      r = @db.search_namespaces_get_unique_id(key, *names)
+      if r.empty? then
+        nil
+      else
+        r
+      end
+    end
+
+    def include_in_primary?(key)
+      check_closed?
+      r = @db.search_primary_get_unique_id(key)
+      if r.empty? then
+        nil
+      else
+        r
+      end
+    end
+
+    def namespaces
+      check_closed?
+      r = secondary_namespaces
+      r.unshift primary_namespace
+      r
+    end
+
+    def primary_namespace
+      check_closed?
+      @db.primary.name
+    end
+
+    def secondary_namespaces
+      check_closed?
+      @db.secondary.names
+    end
+
+    def check_consistency
+      check_closed?
+      @db.check_consistency
+    end
+
+    def always_check_consistency=(bool)
+      @db.always_check=(bool)
+    end
+    def always_check_consistency(bool)
+      @db.always_check
+    end
+
+    # private methods
+    def check_closed?
+      @db or raise IOError, 'closed databank'
+    end
+    private :check_closed?
+
+    #########################################################
+
+    class Results < Hash
+
+      def +(a)
+        raise 'argument must be Results class' unless a.is_a?(self.class)
+        res = self.dup
+        res.update(a)
+        res
+      end
+
+      def *(a)
+        raise 'argument must be Results class' unless a.is_a?(self.class)
+        res = self.class.new
+        a.each_key { |x| res.store(x, a[x]) if self[x] }
+        res
+      end
+
+      def to_s
+        self.values.join
+      end
+
+      #alias each_orig each
+      alias each each_value
+      #alias to_a_orig to_a
+      alias to_a values
+
+    end #class Results
+
+    #########################################################
+
+    module DEBUG
+      @@out = STDERR
+      @@flag = nil
+      def self.out=(io)
+        if io then
+          @@out = io
+          @@out = STDERR if io == true
+          @@flag = true
+        else
+          @@out = nil
+          @@flag = nil
+        end
+        @@out
+      end
+      def self.out
+        @@out
+      end
+      def self.print(*arg)
+        @@flag = true if $DEBUG or $VERBOSE
+        @@out.print(*arg) if @@out and @@flag
+      end
+    end #module DEBUG
+
+    #########################################################
+
+    module Template
+      class NameSpace
+        def filename
+          # should be redifined in child class
+          raise NotImplementedError, "should be redefined in child class"
+        end
+
+        def mapping(filename)
+          # should be redifined in child class
+          raise NotImplementedError, "should be redefined in child class"
+          #Flat_1::FlatMappingFile.new(filename)
+        end
+
+        def initialize(dbname, name)
+          @dbname = dbname
+          @name = name.dup
+          @name.freeze
+          @file = mapping(filename)
+        end
+        attr_reader :dbname, :name, :file
+
+        def search(key)
+          @file.open
+          @file.search(key)
+        end
+
+        def close
+          @file.close
+        end
+
+        def include?(key)
+          r = search(key)
+          unless r.empty? then
+            key
+          else
+            nil
+          end
+        end
+      end #class NameSpace
+    end #module Template
+
+    class FileID
+      def self.new_from_string(str)
+        a = str.split("\t", 2)
+        a[1] = a[1].to_i if a[1]
+        self.new(a[0], a[1])
+      end
+
+      def initialize(filename, filesize = nil)
+        @filename = filename
+        @filesize = filesize
+        @io = nil
+      end
+      attr_reader :filename, :filesize
+
+      def check
+        begin
+          fsize = File.size(@filename)
+          r = ( fsize == @filesize)
+        rescue Errno::ENOENT
+          fsize = -1
+          r = nil
+        end
+        DEBUG.print "FileID: File.size(#{@filename.inspect}) = ",
+          fsize, (r ? ' == ' : ' != ') , @filesize,
+          (r ? '' : ' bad!'), "\n"
+        r
+      end
+
+      def recalc
+        @filesize = File.size(@filename)
+      end
+
+      def to_s(i = nil)
+        if i then
+          str = "fileid_#{i}\t"
+        else
+          str = ''
+        end
+        str << "#{@filename}\t#{@filesize}"
+        str
+      end
+
+      def open
+        unless @io then
+          DEBUG.print "FileID: open #{@filename}\n"
+          @io = File.open(@filename, 'rb')
+          true
+        else
+          nil
+        end
+      end
+
+      def close
+        if @io then
+          DEBUG.print "FileID: close #{@filename}\n"
+          @io.close
+          @io = nil
+          nil
+        else
+          true
+        end
+      end
+
+      def seek(*arg)
+        @io.seek(*arg)
+      end
+
+      def read(size)
+        @io.read(size)
+      end
+
+      def get(pos, length)
+        open
+        seek(pos, IO::SEEK_SET)
+        data = read(length)
+        close
+        data
+      end
+    end #class FileID
+
+    class FileIDs < Array
+      def initialize(prefix, hash)
+        @hash = hash
+        @prefix = prefix
+      end
+
+      def [](n)
+        r = super(n)
+        if r then
+          r
+        else
+          data = @hash["#{@prefix}#{n}"]
+          if data then
+            self[n] = data
+          end
+          super(n)
+        end
+      end
+
+      def []=(n, data)
+        if data.is_a?(FileID) then
+          super(n, data)
+        elsif data then
+          super(n, FileID.new_from_string(data))
+        else
+          # data is nil
+          super(n, nil)
+        end
+        self[n]
+      end
+
+      def add(*arg)
+        arg.each do |filename|
+          self << FileID.new(filename)
+        end
+      end
+
+      def cache_all
+        a = @hash.keys.collect do |k|
+          if k =~ /\A#{Regexp.escape(@prefix)}(\d+)/ then
+            $1.to_i
+          else
+            nil
+          end
+        end
+        a.compact!
+        a.each do |i|
+          self[i]
+        end
+        a
+      end
+
+      def each
+        (0...self.size).each do |i|
+          x = self[i]
+          yield(x) if x
+        end
+        self
+      end
+
+      def each_with_index
+        (0...self.size).each do |i|
+          x = self[i]
+          yield(x, i) if x
+        end
+        self
+      end
+
+      def keys
+        self.cache_all
+        a = []
+        (0...self.size).each do |i|
+          a << i if self[i]
+        end
+        a
+      end
+
+      def filenames
+        self.cache_all
+        a = []
+        self.each do |x|
+          a << x.filename
+        end
+        a
+      end
+
+      def check_all
+        self.cache_all
+        r = true
+        self.each do |x|
+          r = x.check
+          break unless r
+        end
+        r
+      end
+      alias check check_all
+
+      def close_all
+        self.each do |x|
+          x.close
+        end
+        nil
+      end
+      alias close close_all
+
+      def recalc_all
+        self.cache_all
+        self.each do |x|
+          x.recalc
+        end
+        true
+      end
+      alias recalc recalc_all
+
+    end #class FileIDs
+
+    module Flat_1
+      class Record
+        def initialize(str, size = nil)
+          a = str.split("\t")
+          a.each { |x| x.to_s.gsub!(/[\000 ]+\z/, '') }
+          @key = a.shift.to_s
+          @val = a
+          @size = (size or str.length)
+          #DEBUG.print "key=#{@key.inspect},val=#{@val.inspect},size=#{@size}\n"
+        end
+        attr_reader :key, :val, :size
+
+        def to_s
+          self.class.to_string(@size, @key, @val)
+        end
+
+        def self.to_string(size, key, val)
+          sprintf("%-*s", size, key + "\t" + val.join("\t"))
+        end
+
+        def self.create(size, key, val)
+          self.new(self.to_string(size, key, val))
+        end
+
+        def ==(x)
+          self.to_s == x.to_s
+        end
+      end #class Record
+
+      class FlatMappingFile
+        @@recsize_width = 4
+        @@recsize_regex = /\A\d{4}\z/
+
+        def self.open(*arg)
+          self.new(*arg)
+        end
+
+        def initialize(filename, mode = 'rb')
+          @filename = filename
+          @mode = mode
+          @file = nil
+          #@file = File.open(filename, mode)
+          @record_size = nil
+          @records = nil
+        end
+        attr_accessor :mode
+        attr_reader :filename
+        
+        def open
+          unless @file then
+            DEBUG.print "FlatMappingFile: open #{@filename}\n"
+            @file = File.open(@filename, @mode)
+            true
+          else
+            nil
+          end
+        end
+
+        def close
+          if @file then
+            DEBUG.print "FlatMappingFile: close #{@filename}\n"
+            @file.close
+            @file = nil
+          end
+          nil
+        end
+
+        def record_size
+          unless @record_size then
+            open
+            @file.seek(0, IO::SEEK_SET)
+            s = @file.read(@@recsize_width)
+            raise 'strange record size' unless s =~ @@recsize_regex
+            @record_size = s.to_i
+            DEBUG.print "FlatMappingFile: record_size: #{@record_size}\n"
+          end
+          @record_size
+        end
+
+        def get_record(i)
+          rs = record_size
+          seek(i)
+          str = @file.read(rs)
+          #DEBUG.print "get_record(#{i})=#{str.inspect}\n"
+          str
+        end
+
+        def seek(i)
+          rs = record_size
+          @file.seek(@@recsize_width + rs * i)
+        end
+
+        def records
+          unless @records then
+            rs = record_size
+            @records = (@file.stat.size - @@recsize_width) / rs
+            DEBUG.print "FlatMappingFile: records: #{@records}\n"
+          end
+          @records
+        end
+        alias size records
+
+        # methods for writing file
+        def write_record(str)
+          rs = record_size
+          rec = sprintf("%-*s", rs, str)[0..rs]
+          @file.write(rec)
+        end
+
+        def add_record(str)
+          n = records
+          rs = record_size
+          @file.seek(0, IO::SEEK_END)
+          write_record(str)
+          @records += 1
+        end
+
+        def put_record(i, str)
+          n = records
+          rs = record_size
+          if i >= n then
+            @file.seek(0, IO::SEEK_END)
+            @file.write(sprintf("%-*s", rs, '') * (i - n))
+            @records = i + 1
+          else
+            seek(i)
+          end
+          write_record(str)
+        end
+
+        def init(rs)
+          unless 0 < rs and rs < 10 ** @@recsize_width then
+            raise 'record size out of range'
+          end
+          open
+          @record_size = rs
+          str = sprintf("%0*d", @@recsize_width, rs)
+          @file.truncate(0)
+          @file.seek(0, IO::SEEK_SET)
+          @file.write(str)
+          @records = 0
+        end
+
+        # export/import/edit data
+        def each
+          n = records
+          seek(0)
+          (0...n).each do |i|
+            yield Record.new(get_record(i))
+          end
+          self
+        end
+
+        def export_tsv(stream)
+          self.each do |x|
+            stream << "#{x.to_s}\n"
+          end
+          stream
+        end
+
+        def init_with_sorted_tsv_file(filename, flag_primary = false)
+          rec_size = 1
+          f = File.open(filename)
+          f.each do |y|
+            rec_size = y.chomp.length if rec_size < y.chomp.length
+          end
+          self.init(rec_size)
+
+          prev = nil
+          f.rewind
+          if flag_primary then
+            f.each do |y|
+              x = Record.new(y.chomp, rec_size)
+              if prev then
+                if x.key == prev.key
+                  DEBUG.print "Warining: overwrote unique id #{x.key.inspect}\n"
+                else
+                  self.add_record(prev.to_s)
+                end
+              end
+              prev = x
+            end
+            self.add_record(prev.to_s) if prev
+          else
+            f.each do |y|
+              x = Record.new(y.chomp, rec_size)
+              self.add_record(x.to_s) if x != prev
+              prev = x
+            end
+          end
+          f.close
+          self
+        end
+
+        def self.external_sort_proc(sort_program = '/usr/bin/sort')
+          Proc.new do |out, in1, *files|
+            system(sort_program, '-o', out, in1, *files)
+          end
+        end
+
+        def self.external_merge_sort_proc(sort_program = '/usr/bin/sort')
+          Proc.new do |out, in1, *files|
+            # (in1 may be sorted)
+            tf_all = []
+            tfn_all = []
+            files.each do |fn|
+              tf = Tempfile.open('sort')
+              tf.close(false)
+              system(sort_program, '-o', tf.path, fn)
+              tf_all << tf
+              tfn_all << tf.path
+            end
+            system(sort_program, '-m', '-o', out, in1, *tfn_all)
+            tf_all.each do |tf|
+              tf.close(true)
+            end
+          end
+        end
+
+        def self.external_merge_proc(sort_program = '/usr/bin/sort')
+          Proc.new do |out, in1, *files|
+            # files (and in1) must be sorted
+            system(sort_program, '-m', '-o', out, in1, *files)
+          end
+        end
+
+        def self.internal_sort_proc
+          Proc.new do |out, in1, *files|
+            a = IO.readlines(in1)
+            files.each do |fn|
+              IO.foreach(fn) do |x|
+                a << x
+              end
+            end
+            a.sort!
+            of = File.open(out, 'w')
+            a.each { |x| of << x }
+            of.close
+          end
+        end
+
+        def import_tsv_files(flag_primary, mode, sort_proc, *files)
+          require 'tempfile'
+
+          tmpfile1 = Tempfile.open('flat')
+          self.export_tsv(tmpfile1) unless mode == :new
+          tmpfile1.close(false)
+
+          tmpfile0 = Tempfile.open('sorted')
+          tmpfile0.close(false)
+
+          sort_proc.call(tmpfile0.path, tmpfile1.path, *files)
+
+          tmpmap = self.class.new(self.filename + ".#{$$}.tmp~", 'wb+')
+          tmpmap.init_with_sorted_tsv_file(tmpfile0.path, flag_primary)
+          tmpmap.close
+          self.close
+
+          begin
+            File.rename(self.filename, self.filename + ".#{$$}.bak~")
+          rescue Errno::ENOENT
+          end
+          File.rename(tmpmap.filename, self.filename)
+          begin
+            File.delete(self.filename + ".#{$$}.bak~")
+          rescue Errno::ENOENT
+          end
+
+          tmpfile0.close(true)
+          tmpfile1.close(true)
+          self
+        end
+
+
+        # methods for searching
+        def search(key)
+          n = records
+          return [] if n <= 0
+          i = n / 2
+          i_prev = nil
+          DEBUG.print "binary search starts...\n"
+          begin
+            rec = Record.new(get_record(i))
+            i_prev = i
+            if key < rec.key then
+              n = i
+              i = i / 2
+            elsif key > rec.key then
+              i = (i + n) / 2
+            else # key == rec.key
+              result = [ rec.val ]
+              j = i - 1
+              while j >= 0 and
+                  (rec = Record.new(get_record(j))).key == key
+                result << rec.val
+                j = j - 1
+              end
+              result.reverse!
+              j = i + 1
+              while j < n and
+                  (rec = Record.new(get_record(j))).key == key
+                result << rec.val
+                j = j + 1
+              end
+              DEBUG.print "#{result.size} hits found!!\n"
+              return result
+            end
+          end until i_prev == i
+          DEBUG.print "no hits found\n"
+          #nil
+          []
+        end
+      end #class FlatMappingFile
+
+      class PrimaryNameSpace < Template::NameSpace
+        def mapping(filename)
+          FlatMappingFile.new(filename)
+        end
+        def filename
+          File.join(dbname, "key_#{name}.key")
+        end
+      end #class PrimaryNameSpace
+
+      class SecondaryNameSpace < Template::NameSpace
+        def mapping(filename)
+          FlatMappingFile.new(filename)
+        end
+        def filename
+          File.join(dbname, "id_#{name}.index")
+        end
+        def search(key)
+          r = super(key)
+          file.close
+          r.flatten!
+          r
+        end
+      end #class SecondaryNameSpace
+    end #module Flat_1
+
+
+    class NameSpaces < Hash
+      def initialize(dbname, nsclass, arg)
+        @dbname = dbname
+        @nsclass = nsclass
+        if arg.is_a?(String) then
+          a = arg.split("\t")
+        else
+          a = arg
+        end
+        a.each do |x|
+          self[x] = @nsclass.new(@dbname, x)
+        end
+        self
+      end
+
+      def each_names
+        self.names.each do |x|
+          yield x
+        end
+      end
+
+      def each_files
+        self.values.each do |x|
+          yield x
+        end
+      end
+
+      def names
+        keys
+      end
+
+      def close_all
+        values.each { |x| x.file.close }
+      end
+      alias close close_all
+
+      def search(key)
+        r = []
+        values.each do |ns|
+          r.concat ns.search(key)
+        end
+        r.sort!
+        r.uniq!
+        r
+      end
+
+      def search_names(key, *names)
+        r = []
+        names.each do |x|
+          ns = self[x]
+          raise "undefined namespace #{x.inspect}" unless ns
+          r.concat ns.search(key)
+        end
+        r
+      end
+
+      def to_s
+        names.join("\t")
+      end
+    end #class NameSpaces
+
+    class DataBank
+      def self.file2hash(fileobj)
+        hash = {}
+        fileobj.each do |line|
+          line.chomp!
+          a = line.split("\t", 2)
+          hash[a[0]] = a[1]
+        end
+        hash
+      end
+      private_class_method :file2hash
+
+      def self.filename(dbname)
+        File.join(dbname, 'config.dat')
+      end
+
+      def self.read(name, mode = 'rb', *bdbarg)
+        f = File.open(filename(name), mode)
+        hash = file2hash(f)
+        f.close
+        db = self.new(name, nil, hash)
+        db.bdb_open(*bdbarg)
+        db
+      end
+
+      def self.open(*arg)
+        self.read(*arg)
+      end
+
+      def initialize(name, idx_type = nil, hash = {})
+        @dbname = name.dup
+        @dbname.freeze
+        @bdb = nil
+
+        @always_check = true
+        self.index_type = (hash['index'] or idx_type)
+
+        if @bdb then
+          @config = BDBwrapper.new(@dbname, 'config')
+          @bdb_fileids = BDBwrapper.new(@dbname, 'fileids')
+          @nsclass_pri = BDB_1::PrimaryNameSpace
+          @nsclass_sec = BDB_1::SecondaryNameSpace
+        else
+          @config = hash
+          @nsclass_pri = Flat_1::PrimaryNameSpace
+          @nsclass_sec = Flat_1::SecondaryNameSpace
+        end
+        true
+      end
+
+      attr_reader :dbname, :index_type
+
+      def index_type=(str)
+        case str
+        when MAGIC_BDB
+          @index_type = MAGIC_BDB
+          @bdb = true
+          unless defined?(BDB)
+            raise RuntimeError, "Berkeley DB support not found"
+          end
+        when MAGIC_FLAT, '', nil, false
+          @index_type = MAGIC_FLAT
+          @bdb = false
+        else
+          raise 'unknown or unsupported index type'
+        end
+      end
+
+      def to_s
+        a = ""
+        a << "index\t#{@index_type}\n"
+
+        unless @bdb then
+          a << "format\t#{@format}\n"
+          @fileids.each_with_index do |x, i|
+            a << "#{x.to_s(i)}\n"
+          end
+          a << "primary_namespace\t#{@primary.name}\n"
+          a << "secondary_namespaces\t"
+          a << @secondary.names.join("\t")
+          a << "\n"
+        end
+        a
+      end
+
+      def bdb_open(*bdbarg)
+        if @bdb then
+          @config.close
+          @config.open(*bdbarg)
+          @bdb_fileids.close
+          @bdb_fileids.open(*bdbarg)
+          true
+        else
+          nil
+        end
+      end
+
+      def write(mode = 'wb', *bdbarg)
+        unless FileTest.directory?(@dbname) then
+          Dir.mkdir(@dbname)
+        end
+        f = File.open(self.class.filename(@dbname), mode)
+        f.write self.to_s
+        f.close
+
+        if @bdb then
+          bdb_open(*bdbarg)
+          @config['format'] = format
+          @config['primary_namespace'] = @primary.name
+          @config['secondary_namespaces'] = @secondary.names.join("\t")
+          @bdb_fileids.writeback_array('', fileids, *bdbarg)
+        end
+        true
+      end
+
+      def close
+        DEBUG.print "DataBank: close #{@dbname}\n"
+        primary.close
+        secondary.close
+        fileids.close
+        if @bdb then
+          @config.close
+          @bdb_fileids.close
+        end
+        nil
+      end
+
+      ##parameters
+      def primary
+        unless @primary then
+          self.primary = @config['primary_namespace']
+        end
+        @primary
+      end
+
+      def primary=(pri_name)
+        if !pri_name or pri_name.empty? then
+          pri_name = 'UNIQUE'
+        end
+        @primary = @nsclass_pri.new(@dbname, pri_name)
+        @primary
+      end
+
+      def secondary
+        unless @secondary then
+          self.secondary = @config['secondary_namespaces']
+        end
+        @secondary
+      end
+
+      def secondary=(sec_names)
+        if !sec_names then
+          sec_names = []
+        end
+        @secondary = NameSpaces.new(@dbname, @nsclass_sec, sec_names)
+        @secondary
+      end
+
+      def format=(str)
+        @format = str.to_s.dup
+      end
+
+      def format
+        unless @format then
+          self.format = @config['format']
+        end
+        @format
+      end
+
+      def fileids
+        unless @fileids then
+          init_fileids
+        end
+        @fileids
+      end
+
+      def init_fileids
+        if @bdb then
+          @fileids = FileIDs.new('', @bdb_fileids)
+        else
+          @fileids = FileIDs.new('fileid_', @config)
+        end
+        @fileids
+      end
+
+      # high level methods
+      def always_check=(bool)
+        if bool then
+          @always_check = true
+        else
+          @always_check = false
+        end
+      end
+      attr_reader :always_check
+
+      def get_flatfile_data(f, pos, length)
+        fi = fileids[f.to_i]
+        if @always_check then
+          raise "flatfile #{fi.filename.inspect} may be modified" unless fi.check
+        end
+        fi.get(pos.to_i, length.to_i)
+      end
+
+      def search_all_get_unique_id(key)
+        s = secondary.search(key)
+        p = primary.include?(key)
+        s.push p if p
+        s.sort!
+        s.uniq!
+        s
+      end
+
+      def search_primary(*arg)
+        r = Results.new
+        arg.each do |x|
+          a = primary.search(x)
+          # a is empty or a.size==1 because primary key must be unique
+          r.store(x, get_flatfile_data(*a[0])) unless a.empty?
+        end
+        r
+      end
+
+      def search_all(key)
+        s = search_all_get_unique_id(key)
+        search_primary(*s)
+      end
+
+      def search_primary_get_unique_id(key)
+        s = []
+        p = primary.include?(key)
+        s.push p if p
+        s
+      end
+
+      def search_namespaces_get_unique_id(key, *names)
+        if names.include?(primary.name) then
+          n2 = names.dup
+          n2.delete(primary.name)
+          p = primary.include?(key)
+        else
+          n2 = names
+          p = nil
+        end
+        s = secondary.search_names(key, *n2)
+        s.push p if p
+        s.sort!
+        s.uniq!
+        s
+      end
+
+      def search_namespaces(key, *names)
+        s = search_namespaces_get_unique_id(key, *names)
+        search_primary(*s)
+      end
+
+      def check_consistency
+        fileids.check_all
+      end
+    end #class DataBank
+
+  end #class FlatFileIndex
+end #module Bio
+
+######################################################################
+
+=begin
+
+= Bio::FlatFileIndex
+
+--- Bio::FlatFileIndex.new(dbname)
+--- Bio::FlatFileIndex.open(dbname)
+
+      Opens existing databank. Databank is a directory which contains
+      indexed files and configuration files. The type of the databank
+      (flat or BerkeleyDB) are determined automatically.
+
+--- Bio::FlatFileIndex#close
+
+      Closes opened databank.
+
+--- Bio::FlatFileIndex#closed?
+
+      Returns true if already closed. Otherwise, returns false.
+
+--- Bio::FlatFileIndex#get_by_id(key)
+
+      Common interface defined in registry.rb.
+      Searching databank and returns entry (or entries) as a string.
+      Multiple entries (contatinated to one string) may be returned.
+      Returns empty string If not found.
+
+--- Bio::FlatFileIndex#search(key)
+
+      Searching databank and returns a Bio::FlatFileIndex::Results object.
+
+--- Bio::FlatFileIndex#include?(key)
+
+      Searching databank.
+      If found, returns an array of unique IDs (primary identifiers).
+      If not found, returns nil.
+
+--- Bio::FlatFileIndex#search_primary(key)
+
+      Searching only primary namespece.
+      Returns a Bio::FlatFileIndex::Results object.
+
+--- Bio::FlatFileIndex#search_namespaces(key, name1, name2, ...)
+
+      Searching only specific namespeces.
+      Returns a Bio::FlatFileIndex::Results object.
+
+--- Bio::FlatFileIndex#include_in_primary?(key)
+
+      Same as #include?, but serching only primary namespace.
+
+--- Bio::FlatFileIndex#include_in_namespaces?(key, name1, name2, ...)
+
+      Same as #include?, but serching only specific namespaces.
+
+--- Bio::FlatFileIndex#namespaces
+
+      Returns names of namespaces defined in the databank.
+      (example: [ 'LOCUS', 'ACCESSION', 'VERSION' ] )
+
+--- Bio::FlatFileIndex#primary_namespace
+
+      Returns name of primary namespace.
+
+--- Bio::FlatFileIndex#secondary_namespaces
+
+      Returns names of secondary namespaces.
+
+--- Bio::FlatFileIndex#default_namespaces= [ str1, str2, ... ]
+--- Bio::FlatFileIndex#default_namespaces= nil
+
+      Set default namespaces.
+      nil means all namespaces in the databank.
+      Default namespaces specified in this method only affect 
+      #get_by_id, #search, and #include? methods.
+      Default of default namespaces is nil (that is, all namespaces
+      are search destinations by default).
+
+--- Bio::FlatFileIndex#default_namespaces
+
+      Returns default namespaces.
+      nil means all namespaces.
+
+--- Bio::FlatFileIndex#check_consistency
+
+      Raise RuntimeError if flatfiles are changed after creating
+      the databank. (This check only compare file sizes as
+      described in the OBDA specification.)
+
+--- Bio::FlatFileIndex#always_check_consistency=(bool)
+--- Bio::FlatFileIndex#always_check_consistency
+
+      If true, consistency checks are performed every time
+      accessing flatfiles. If nil/false, no checks are performed.
+      Default of always_check_consistency is true.
+
+== Bio::FlatFileIndex::Results
+
+      This object is made by Bio::FlatFileIndex methods.
+      Currently, this class inherits Hash, but internal
+      structure of this class may be changed anytime.
+      Only using methods described below are strongly recomended.
+
+--- Bio::FlatFileIndex::Results#to_a
+
+      Returns an array of strings.
+      If no search results are exist, returns an empty array.
+
+--- Bio::FlatFileIndex::Results#each
+
+      Iterates over each result(string).
+      Same as to_a.each.
+
+--- Bio::FlatFileIndex::Results#to_s
+
+      Returns a string. (concatinated if multiple results exists).
+      Same as to_a.join('').
+
+--- Bio::FlatFileIndex::Results#size
+
+      Returns number of results.
+      Same as to_a.size.
+
+--- Bio::FlatFileIndex::Results#+(res)
+
+      Add search results.
+      "a + b" means "a OR b".
+      * Example
+          # I want to search 'ADH_IRON_1' OR 'ADH_IRON_2'
+          db = Bio::FlatFIleIndex.new(location)
+          a1 = db.search('ADH_IRON_1')
+          a2 = db.search('ADH_IRON_2')
+          # a1 and a2 are Bio::FlatFileIndex::Results objects.
+          print a1 + a2
+
+--- Bio::FlatFileIndex::Results#*(res)
+
+      Returns set intersection of results.
+      "a * b" means "a AND b".
+      * Example
+          # I want to search 'HIS_KIN' AND 'human'
+          db = Bio::FlatFIleIndex.new(location)
+          hk = db.search('HIS_KIN')
+          hu = db.search('human')
+          # hk and hu are Bio::FlatFileIndex::Results objects.
+          print hk * hu
+
+== Bio::FlatFileIndex::DEBUG
+
+      Module for output debug messages.
+      Default setting: If $DEBUG or $VERBOSE is true, output debug
+      messages to STDERR; Otherwise, don't output messages.
+
+--- Bio::FlatFileIndex::DEBUG.out=(io)
+
+      Set debug messages output destination.
+      If true is given, outputs to STDERR.
+      If nil is given, outputs nothing.
+      This method affects ALL of FlatFileIndex related objects/methods.
+
+== Other classes/modules
+
+      Classes/modules not described in this file are internal use only.
+
+== SEE ALSO
+
+* ((<URL:http://obda.open-bio.org/>))
+* ((<URL:http://cvs.open-bio.org/cgi-bin/viewcvs/viewcvs.cgi/obda-specs/?cvsroot=obf-common>))
+
+=end