athena 0.0.2.56 → 0.0.3.58

data/README

@@ -2,7 +2,7 @@
 
 == VERSION
 
-This documentation refers to athena version 0.0.2
+This documentation refers to athena version 0.0.3
 
 
 == DESCRIPTION
@@ -17,7 +17,7 @@ TODO: well, the description... ;-)
 
 == LICENSE AND COPYRIGHT
 
-Copyright (C) 2007 University of Cologne,
+Copyright (C) 2007-2008 University of Cologne,
 Albertus-Magnus-Platz, 50932 Cologne, Germany
 
 athena is free software: you can redistribute it and/or modify it under the

data/Rakefile

@@ -1,21 +1,22 @@
+$:.unshift('lib')
+require 'athena'
+
 begin
   require 'hen'
+
+  Hen.lay! {{
+    :rubyforge => {
+      :package => 'athena'
+    },
+
+    :gem => {
+      :version      => Athena::VERSION,
+      :summary      => 'Convert database files to various formats.',
+      :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+      :extra_files  => FileList['[A-Z]*', 'example/*'].to_a,
+      :dependencies     => %w[xmlstreamin ruby-nuggets]
+    }
+  }}
 rescue LoadError
   abort "Please install the 'hen' gem first."
 end
-
-require 'lib/athena/version'
-
-Hen.lay! {{
-  :rubyforge => {
-    :package => 'athena'
-  },
-
-  :gem => {
-    :version      => Athena::VERSION,
-    :summary      => 'Convert database files to various formats.',
-    :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
-    :extra_files  => FileList['[A-Z]*', 'example/*'].to_a,
-    :dependencies     => %w[xmlstreamin ruby-nuggets]
-  }
-}}

data/bin/athena

@@ -5,9 +5,9 @@
 #                                                                             #
 # athena -- Convert database files to various formats                         #
 #                                                                             #
-# Copyright (C) 2007 University of Cologne,                                   #
-#                    Albertus-Magnus-Platz,                                   #
-#                    50932 Cologne, Germany                                   #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
 #                                                                             #
 # Authors:                                                                    #
 #     Jens Wille <jens.wille@uni-koeln.de>                                    #
@@ -65,11 +65,11 @@ OptionParser.new { |opts|
   opts.on('-i', '--input FILE',  "Input file [Default: STDIN]") { |f|
     abort "Can't find input file: #{f}." unless File.readable?(f)
 
-    options[:input] = File.open(f, 'r')
+    options[:input] = File.directory?(f) ? Dir.open(f) : File.open(f, 'r')
 
     p = File.basename(f).split('.')
     options[:spec_fallback]   = p.last.downcase
-    options[:target_fallback] = p[0..-2].join('.')
+    options[:target_fallback] = p.size > 1 ? p[0..-2].join('.') : p.first
   }
 
   opts.on('-s', '--spec SPEC', "Input format (spec) [Default: file ending of <input-file>]") { |s|

data/lib/athena.rb

@@ -3,9 +3,9 @@
 #                                                                             #
 # athena -- Convert database files to various formats                         #
 #                                                                             #
-# Copyright (C) 2007 University of Cologne,                                   #
-#                    Albertus-Magnus-Platz,                                   #
-#                    50932 Cologne, Germany                                   #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
 #                                                                             #
 # Authors:                                                                    #
 #     Jens Wille <jens.wille@uni-koeln.de>                                    #
@@ -35,6 +35,9 @@
 # class method _convert_ supplied. This way, a specific format can even function
 # as both input and output format.
 
+module Athena
+end
+
 require 'athena/util'
 require 'athena/parser'
 require 'athena/record'

data/lib/athena/formats.rb

@@ -3,9 +3,9 @@
 #                                                                             #
 # A component of athena, the database file converter.                         #
 #                                                                             #
-# Copyright (C) 2007 University of Cologne,                                   #
-#                    Albertus-Magnus-Platz,                                   #
-#                    50932 Cologne, Germany                                   #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
 #                                                                             #
 # Authors:                                                                    #
 #     Jens Wille <jens.wille@uni-koeln.de>                                    #
@@ -26,65 +26,61 @@
 ###############################################################################
 #++
 
-module Athena
+class Athena::Formats
 
-  class Formats
+  @formats = { :in => {}, :out => {} }
 
-    @formats = { :in => {}, :out => {} }
+  class << self
 
-    class << self
-
-      def formats
-        Formats.instance_variable_get(:@formats)
-      end
-
-      def [](direction, format)
-        formats[direction][format]
-      end
-
-      def valid_format?(direction, format)
-        formats[direction].has_key?(format)
-      end
+    def formats
+      Athena::Formats.instance_variable_get(:@formats)
+    end
 
-      def deferred?
-        false
-      end
+    def [](direction, format)
+      formats[direction][format]
+    end
 
-      def convert(*args)
-        raise NotImplementedError, 'must be defined by sub-class'
-      end
+    def valid_format?(direction, format)
+      formats[direction].has_key?(format)
+    end
 
-      private
+    def deferred?
+      false
+    end
 
-      def register_format(direction, format)
-        if existing = formats[direction][format]
-          raise DuplicateFormatDefinitionError,
-            "format already defined (#{direction}): #{format} = #{existing}"
-        end
+    def convert(*args)
+      raise NotImplementedError, 'must be defined by sub-class'
+    end
 
-        formats[direction][format] = self
-      end
+    private
 
-      def register_formats(direction, *formats)
-        formats.each { |format|
-          register_format(direction, format)
-        }
+    def register_format(direction, format)
+      if existing = formats[direction][format]
+        raise DuplicateFormatDefinitionError,
+          "format already defined (#{direction}): #{format} = #{existing}"
       end
 
+      formats[direction][format] = self
     end
 
-    def parse(*args)
-      raise NotImplementedError, 'must be defined by sub-class'
+    def register_formats(direction, *formats)
+      formats.each { |format|
+        register_format(direction, format)
+      }
     end
 
-    class DuplicateFormatDefinitionError < StandardError
-    end
+  end
 
-    class FormatArgumentError < ArgumentError
-    end
+  def parse(*args)
+    raise NotImplementedError, 'must be defined by sub-class'
+  end
 
+  class DuplicateFormatDefinitionError < StandardError
   end
-  
+
+  class FormatArgumentError < ArgumentError
+  end
+
 end
 
 Dir[__FILE__.sub(/\.rb$/, '/**/*.rb')].each { |rb|

data/lib/athena/formats/dbm.rb

@@ -3,9 +3,9 @@
 #                                                                             #
 # A component of athena, the database file converter.                         #
 #                                                                             #
-# Copyright (C) 2007 University of Cologne,                                   #
-#                    Albertus-Magnus-Platz,                                   #
-#                    50932 Cologne, Germany                                   #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
 #                                                                             #
 # Authors:                                                                    #
 #     Jens Wille <jens.wille@uni-koeln.de>                                    #
@@ -28,41 +28,37 @@
 
 require 'iconv'
 
-module Athena
+class Athena::Formats
 
-  class Formats
+  class DBM < Athena::Formats
 
-    class DBM < Athena::Formats
+    register_formats :out, 'dbm', 'midos'
 
-      register_formats :out, 'dbm', 'midos'
+    CRLF = "\015\012"
 
-      CRLF = "\015\012"
+    ICONV_TO_LATIN1 = Iconv.new('latin1', 'utf-8')
 
-      ICONV_TO_LATIN1 = Iconv.new('latin1', 'utf-8')
+    VALUE_SEPARATOR  = '|'
+    RECORD_SEPARATOR = '&&&'
 
-      VALUE_SEPARATOR  = '|'
-      RECORD_SEPARATOR = '&&&'
+    def self.convert(record)
+      dbm = ["ID:#{record.id}"]
 
-      def self.convert(record)
-        dbm = ["ID:#{record.id}"]
+      record.struct.each { |field, struct|
+        strings = struct[:elements].inject([]) { |array, element|
+          values = (struct[:values][element] || []).map { |v|
+            (v || '').strip.gsub(/(?:\r?\n)+/, ' ')
+          }.reject { |v| v.empty? }
 
-        record.struct.each { |field, struct|
-          strings = struct[:elements].inject([]) { |array, element|
-            values = (struct[:values][element] || []).map { |v|
-              (v || '').strip.gsub(/(?:\r?\n)+/, ' ')
-            }.reject { |v| v.empty? }
-
-            array << (values.empty? ? struct[:empty] : values.join(VALUE_SEPARATOR))
-          }
-
-          dbm << "#{field.to_s.upcase}:#{ICONV_TO_LATIN1.iconv(struct[:string] % strings)}"
+          array << (values.empty? ? struct[:empty] : values.join(VALUE_SEPARATOR))
         }
 
-        dbm << RECORD_SEPARATOR
+        dbm << "#{field.to_s.upcase}:#{ICONV_TO_LATIN1.iconv(struct[:string] % strings)}"
+      }
 
-        dbm.join(CRLF) << CRLF << CRLF
-      end
+      dbm << RECORD_SEPARATOR
 
+      dbm.join(CRLF) << CRLF << CRLF
     end
 
   end

data/lib/athena/formats/ferret.rb

@@ -0,0 +1,91 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of athena, the database file converter.                         #
+#                                                                             #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# athena is free software; you can redistribute it and/or modify it under the #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# athena is distributed in the hope that it will be useful, but WITHOUT ANY   #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with athena. If not, see <http://www.gnu.org/licenses/>.                    #
+#                                                                             #
+###############################################################################
+#++
+
+require 'rubygems'
+require 'ferret'
+
+class Athena::Formats
+
+  class Ferret < Athena::Formats
+
+    register_format :in, 'ferret'
+
+    attr_reader :record_element, :config, :parser, :match_all_query
+
+    def initialize(parser)
+      config = parser.config.dup
+
+      case @record_element = config.delete(:__record_element)
+        when String
+          # fine!
+        when nil
+          raise NoRecordElementError, 'no record element specified'
+        else
+          raise IllegalRecordElementError, "illegal record element #{@record_element}"
+      end
+
+      @config = config
+      @parser = parser
+
+      @match_all_query = ::Ferret::Search::MatchAllQuery.new
+    end
+
+    def parse(source)
+      search_all(source) { |doc|
+        record = Athena::Record.new(parser.block, doc[record_element])
+
+        config.each { |element, field_config|
+          record.update(element, doc[element], field_config)
+        }
+
+        record.close
+      }
+    end
+
+    private
+
+    def search_all(source)
+      index = ::Ferret::Index::Index.new(
+        :path              => source.path,
+        :create_if_missing => false
+      ).searcher
+
+      index.search_each(match_all_query, :limit => :all) { |doc_id, _|
+        yield index[doc_id] if block_given?
+      }
+    end
+
+    class NoRecordElementError < StandardError
+    end
+
+    class IllegalRecordElementError < StandardError
+    end
+
+  end
+
+end

data/lib/athena/formats/lingo.rb

@@ -3,9 +3,9 @@
 #                                                                             #
 # A component of athena, the database file converter.                         #
 #                                                                             #
-# Copyright (C) 2007 University of Cologne,                                   #
-#                    Albertus-Magnus-Platz,                                   #
-#                    50932 Cologne, Germany                                   #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
 #                                                                             #
 # Authors:                                                                    #
 #     Jens Wille <jens.wille@uni-koeln.de>                                    #
@@ -29,113 +29,109 @@
 require 'iconv'
 require 'enumerator'
 
-module Athena
+class Athena::Formats
 
-  class Formats
+  module Lingo
 
-    module Lingo
+    class Base < Athena::Formats
 
-      class Base < Athena::Formats
+      class << self
 
-        class << self
-
-          def convert(record)
-            record.struct.inject([]) { |terms, (field, struct)|
-              terms << struct[:elements].inject([]) { |array, element|
-                array += (struct[:values][element] || []).map { |v|
-                  (v || '').strip.gsub(/(?:\r?\n)+/, ' ')
-                }.reject { |v| v.empty? }
-              }
+        def convert(record)
+          record.struct.inject([]) { |terms, (field, struct)|
+            terms << struct[:elements].inject([]) { |array, element|
+              array += (struct[:values][element] || []).map { |v|
+                (v || '').strip.gsub(/(?:\r?\n)+/, ' ')
+              }.reject { |v| v.empty? }
             }
-          end
-
-          def deferred?
-            true
-          end
+          }
+        end
 
-          private
+        def deferred?
+          true
+        end
 
-          def check_number_of_arguments(expected, actual, blow = false, &block)
-            return true if block ? block[actual] : expected == actual
+        private
 
-            msg = "wrong number of arguments for #{self} (#{actual} for #{expected})"
+        def check_number_of_arguments(expected, actual, blow = false, &block)
+          return true if block ? block[actual] : expected == actual
 
-            if blow
-              raise FormatArgumentError, msg
-            else
-              warn msg
-              return false
-            end
-          end
+          msg = "wrong number of arguments for #{self} (#{actual} for #{expected})"
 
-          def check_number_of_arguments!(expected, actual, &block)
-            check_number_of_arguments(expected, actual, true, &block)
+          if blow
+            raise FormatArgumentError, msg
+          else
+            warn msg
+            return false
           end
+        end
 
+        def check_number_of_arguments!(expected, actual, &block)
+          check_number_of_arguments(expected, actual, true, &block)
         end
 
       end
 
-      # "Nasenbär\n"
-      class SingleWord < Athena::Formats::Lingo::Base
+    end
 
-        register_formats :out, 'lingo/single_word'
+    # "Nasenbär\n"
+    class SingleWord < Athena::Formats::Lingo::Base
 
-        def self.convert(record)
-          super.flatten
-        end
+      register_formats :out, 'lingo/single_word'
 
+      def self.convert(record)
+        super.flatten
       end
 
-      # "John Vorhauer*Vorhauer, John\n"
-      class KeyValue < Athena::Formats::Lingo::Base
+    end
 
-        register_formats :out, 'lingo/key_value'
+    # "John Vorhauer*Vorhauer, John\n"
+    class KeyValue < Athena::Formats::Lingo::Base
 
-        def self.convert(record)
-          super.map { |terms|
-            next unless check_number_of_arguments(2, terms.size)
+      register_formats :out, 'lingo/key_value'
 
-            terms.join('*')
-          }.compact
-        end
+      def self.convert(record)
+        super.map { |terms|
+          next unless check_number_of_arguments(2, terms.size)
 
+          terms.join('*')
+        }.compact
       end
 
-      # "Essen,essen #v Essen #s Esse #s\n"
-      class WordClass < Athena::Formats::Lingo::Base
+    end
 
-        register_formats :out, 'lingo/word_class'
+    # "Essen,essen #v Essen #s Esse #s\n"
+    class WordClass < Athena::Formats::Lingo::Base
 
-        def self.convert(record)
-          super.map { |terms|
-            next unless check_number_of_arguments('odd, > 1', terms.size) { |actual|
-              actual > 1 && actual % 2 == 1
-            }
+      register_formats :out, 'lingo/word_class'
 
-            [terms.shift, terms.to_enum(:each_slice, 2).map { |form, wc|
-              "#{form} ##{wc}"
-            }.join(' ')].join(',')
-          }.compact
-        end
+      def self.convert(record)
+        super.map { |terms|
+          next unless check_number_of_arguments('odd, > 1', terms.size) { |actual|
+            actual > 1 && actual % 2 == 1
+          }
 
+          [terms.shift, terms.to_enum(:each_slice, 2).map { |form, wc|
+            "#{form} ##{wc}"
+          }.join(' ')].join(',')
+        }.compact
       end
 
-      # "Fax;Faxkopie;Telefax\n"
-      class MultiValue < Athena::Formats::Lingo::Base
+    end
 
-        register_formats :out, 'lingo/multi_value', 'lingo/multi_key'
+    # "Fax;Faxkopie;Telefax\n"
+    class MultiValue < Athena::Formats::Lingo::Base
 
-        def self.convert(record)
-          super.map { |terms|
-            next unless check_number_of_arguments('> 1', terms.size) { |actual|
-              actual > 1
-            }
+      register_formats :out, 'lingo/multi_value', 'lingo/multi_key'
 
-            terms.join(';')
-          }.compact
-        end
+      def self.convert(record)
+        super.map { |terms|
+          next unless check_number_of_arguments('> 1', terms.size) { |actual|
+            actual > 1
+          }
 
+          terms.join(';')
+        }.compact
       end
 
     end