blackwinter-athena 0.0.6

data/README

@@ -0,0 +1,33 @@
+= athena - Convert database files to various formats
+
+== VERSION
+
+This documentation refers to athena version 0.0.6
+
+
+== DESCRIPTION
+
+TODO: well, the description... ;-)
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@uni-koeln.de>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007-2008 University of Cologne,
+Albertus-Magnus-Platz, 50932 Cologne, Germany
+
+athena is free software: you can redistribute it and/or modify it under the
+terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+athena is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+athena. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,22 @@
+$:.unshift('lib')
+require 'athena'
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    :rubyforge => {
+      :package => 'athena'
+    },
+
+    :gem => {
+      :version      => Athena::VERSION,
+      :summary      => 'Convert database files to various formats.',
+      :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+      :extra_files  => FileList['[A-Z]*', 'example/*'].to_a,
+      :dependencies     => %w[xmlstreamin ruby-nuggets]
+    }
+  }}
+rescue LoadError
+  abort "Please install the 'hen' gem first."
+end

data/bin/athena

@@ -0,0 +1,183 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# athena -- Convert database files to various formats                         #
+#                                                                             #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# athena is free software; you can redistribute it and/or modify it under the #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# athena is distributed in the hope that it will be useful, but WITHOUT ANY   #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with athena. If not, see <http://www.gnu.org/licenses/>.                    #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'yaml'
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'athena'
+
+USAGE = "Usage: #{$0} [-h|--help] [options]"
+abort USAGE if ARGV.empty?
+
+# Global variable to handle verbosity
+$Verbose = {}
+
+options = {
+  :config => 'config.yaml',
+  :input  => STDIN,
+  :output => STDOUT,
+  :target => nil
+}
+
+OptionParser.new { |opts|
+  opts.banner = USAGE
+
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-c', '--config YAML', "Config file [Default: #{options[:config]}#{' (currently not present)' unless File.readable?(options[:config])}]") { |f|
+    abort "Can't find config file: #{f}." unless File.readable?(f)
+
+    options[:config] = f
+  }
+
+  opts.separator ''
+
+  opts.on('-i', '--input FILE',  "Input file [Default: STDIN]") { |f|
+    abort "Can't find input file: #{f}." unless File.readable?(f)
+
+    options[:input] = File.directory?(f) ? Dir.open(f) : File.open(f, 'r')
+
+    p = File.basename(f).split('.')
+    options[:spec_fallback]   = p.last.downcase
+    options[:target_fallback] = p.size > 1 ? p[0..-2].join('.') : p.first
+  }
+
+  opts.on('-s', '--spec SPEC', "Input format (spec) [Default: file ending of <input-file>]") { |s|
+    options[:spec] = s.downcase
+  }
+
+  opts.on('-L', '--list-specs', "List available input formats (specs) and exit") {
+    puts "Available input formats (specs):"
+
+    formats = Athena.input_formats
+    max     = formats.map { |a, _| a.length }.max
+    formats.each { |f, k|
+      puts "  - %-#{max}s = %s" % [f, k]
+    }
+
+    exit 0
+  }
+
+  opts.separator ''
+
+  opts.on('-o', '--output FILE',  "Output file [Default: STDOUT]") { |f|
+    options[:output] = File.open(f, 'w')
+
+    options[:format_fallback] = f.split('.').last.downcase
+  }
+
+  opts.on('-f', '--format FORMAT', "Output format [Default: file ending of <output-file>]") { |f|
+    options[:format] = f.downcase
+  }
+
+  opts.on('-l', '--list-formats', "List available output formats and exit") {
+    puts "Available output formats:"
+
+    formats = Athena.output_formats
+    max     = formats.map { |a, _| a.length }.max
+    formats.each { |f, k|
+      puts "  - %-#{max}s = %s" % [f, k]
+    }
+
+    exit 0
+  }
+
+  opts.separator ''
+
+  opts.on('-t', '--target ID', "Target whose config to use [Default: <input-file> minus file ending,", "plus '.<spec>', plus ':<format>' (reversely in turn)]") { |t|
+    options[:target] = t
+  }
+
+  opts.separator ''
+  opts.separator 'Generic options:'
+
+  opts.on('-v', '--verbose [WHAT]', "Be verbose about what's being done. Optional argument is a comma-separated", "list of what should be output, or 'all' [Default: 'all']") { |what|
+    if what.nil? || what == 'all'
+      $Verbose.default = true
+    else
+      what.split(',').each { |w|
+        $Verbose[w.to_sym] = true
+      }
+    end
+  }
+
+  opts.on('-h', '--help', 'Print this help message and exit') {
+    abort opts.to_s
+  }
+
+  opts.on('--version', 'Print program version and exit') {
+    abort "#{File.basename($0)} v#{Athena::VERSION}"
+  }
+}.parse!
+
+spec = options[:spec] || options[:spec_fallback]
+abort "No input format (spec) specified and none could be inferred." unless spec
+abort "Invalid input format (spec): #{spec}. Use '-L' to get a list of available specs." unless Athena.valid_input_format?(spec)
+
+format = options[:format] || options[:format_fallback]
+abort "No output format specified and none could be inferred." unless format
+abort "Invalid output format: #{format}. Use '-l' to get a list of available formats." unless Athena.valid_output_format?(format)
+
+yaml = YAML.load_file(options[:config])
+if t = options[:target]
+  target = t
+  config = yaml[t.to_sym]
+else
+  [options[:target_fallback] || 'generic', ".#{spec}", ":#{format}"].inject([]) { |s, t|
+    s << (s.last ? s.last + t : t)
+  }.reverse.find { |t|
+    target = t
+    config = yaml[t.to_sym]
+  }
+end
+abort "Config not found for target: #{target}." unless config
+
+parser = Athena.parser(config, spec)
+
+if Athena.deferred_output?(format)
+  records = parser.parse(options[:input])
+
+  records.map { |record|
+    record.to(format)
+  }.flatten.sort.uniq.each { |line|
+    options[:output].puts line
+  }
+else
+  records = parser.parse(options[:input]) { |record|
+    options[:output].puts record.to(format)
+  }
+end
+
+Athena::Util.verbose(:count) do
+  spit records.size
+end

data/example/config.yaml

@@ -0,0 +1,72 @@
+:example:
+  :__record_element: "record"
+  :author: "author"
+  :title:
+    :elements:
+      - "title/main"
+      - "title/subtitle"
+    :string: "%s: %s"
+    :empty: ">>n/a<<"
+  :place:
+    :elements:
+      - "city"
+      - "country"
+    :separator: " / "
+:multiple-fields-per-element:
+  :__record_element: "record"
+  :author:
+    :elements:
+      - "author"
+      - "city"
+      - "title/main"
+    :string: "author=%s (city=%s) [title/main=%s]"
+  :title:
+    :elements:
+      - "title/main"
+      - "title/subtitle"
+      - "author"
+    :string: "title/main=%s: title/subtitle=%s (author=%s)"
+    :empty: ">>n/a<<"
+  :place:
+    :elements:
+      - "city"
+      - "country"
+      - "title"
+    :string: "city=%s / country=%s (title=%s)"
+:sisis-ex:
+  :__record_element: "0000"  # KatalogNr
+  :author:
+    :elements:
+      - "0100"               # VerfAnsetz
+      - "0101"               # Verf_Ordn
+    :string: "%s (%s)"
+  :title:
+    :elements:
+      - "0331"               # HST
+      - "0335"               # HSTZusatz
+      - "0370"               # Untertitstab
+    :string: "%s : %s [%s]"
+  :place: "2028"             # ort2sb
+:sisis-multiple-fields-per-element:
+  :__record_element: "0000"
+  :author:
+    :elements:
+      - "0100"
+      - "0101"
+      - "0331"
+      - "2028"
+    :string: "VerfAnsetz=%s (Verf_Ordn=%s) [HST=%s] / ort2sb=%s"
+  :title:
+    :elements:
+      - "0331"
+      - "0335"
+      - "0370"
+      - "0100"
+    :string: "HST=%s : HSTZusatz=%s [Untertitstab=%s] (VerfAnsetz=%s)"
+  :place:
+    :elements:
+      - "2028"
+      - "0335"
+      - "0370"
+      - "0100"
+    :string: "ort2sb=%s (HSTZusatz=%s [Untertitstab=%s] / VerfAnsetz=%s)"

data/example/example.xml

@@ -0,0 +1,26 @@
+<root>
+  <record>
+    <author>
+      <first>John Doe</first>
+      <second>John Q.</second>
+      <third>JJ</third>
+    </author>
+    <title>
+      <main>Just kiddin'</main>
+      <subtitle>heh?</subtitle>
+    </title>
+    <city>Nowhere</city>
+    <country>None</country>
+  </record>
+  <record>
+    <author>
+      Jane Doe
+      <separator />
+      JD
+    </author>
+    <title>
+      <main>No title</main>
+    </title>
+    <city>Nowhere</city>
+  </record>
+</root>

data/example/sisis-ex.txt

@@ -0,0 +1,90 @@
+0000:3
+0001:000000003
+0002:02.05.2001
+0003:27.08.2002
+0015:ger; lat
+0036:m
+0100.001:[Pecka, Michael]
+0110.001:[Pieczek, Michael]
+0331:Denckwürdiges Geheimnuß
+0335:Teutsche Erklärung des Kupfferstücks
+0370.001:Dum tua privato fessus das lumina somno
+0370.002:Einer/ oder gar drey müssen für das Volck sterben
+0424:1620
+0425:[ca. 1620]
+0433:1 Bl.
+0434:1 Ill.; Radierung 19,5 x 27,5 cm(nach einem Kupferstich von Gaspar Dooms)
+0435:Satzspiegel 38 x 30 cm
+0440.001:[S.l.]
+0509:Verfasser ermittelt aus Nachweis
+0511:Erscheinungsjahr ermittelt aus Inhalt und Nachweis
+0527.001:Andere Ausgabe: Pfeffer, Maria
+0720.001:Politisches Flugblatt; Prokaiserliches Flugblatt
+0721.001:Dreissigjähriger Krieg; Böhmisch-Pfälzischer Krieg; Krieg; 1600-1650
+0722.001:Böhmen <Königreich>; Schlesien; Mähren; Lausitz; Bayern; Österreich; Prag
+0723.001:Ferdinand <Römisch-Deutsches Reich, Kaiser, II.>; Christus
+1105.001:Einbl. V,8 b-10
+1105.002:Einbl. V,60
+1125.001:Blattmaß 38 x 30 cm; Hinterklebt ; Klebereste
+1125.002:Blattmaß 30 x 31,5 cm; Fragment: unterer Textteil fehlt teilweise
+1145.001:ei
+1145.002:ei
+2005:Metallschnittrahmen; Metallschnittleiste als Spaltentrenner; Zahlen als Marginalien
+2006:Deutsche illustrierte Flugblätter des 16. und 17. Jahrhunderts . Hrsg. von Wolfgang Harms. Bd. 2
+2008:Denkwürdig Geheimnis Prophezeiung Zustand Böhmen Unwesen
+2018:Lateinischer Text in der Radierung als Beischriften; Im unteren Teil des Bildes Versform
+2023:¤ 61B2(Ferdinand <Römisch-Deutsches Reich, Kaiser, II.>)(+1)
+2028:BLA
+2029:001
+2030.001:http://zoom.bib-bvb.de/StyleServer/calcrgn?cat=einbl&item=/300000183_0_r.sid&wid=750&hei=500&style=bsb/einbl.xsl&plugin=false
+2030.002:http://zoom.bib-bvb.de/StyleServer/calcrgn?cat=einbl&item=/300000368_0_r.sid&wid=750&hei=500&style=bsb/einbl.xsl&plugin=false
+9999:
+0000:4
+0001:000000004
+0002:03.05.2001
+0003:29.08.2002
+0015:ger
+0036:m
+0331:Machometische Zanck= und Haderkatzen
+0370.001:UNser Planet Ist Machomet, Saturnisch Thier Sind wir allhier
+0424:1621
+0425:1621
+0433:1 Bl.
+0434:1 Ill.; Kupferstich 14 x 24 cm
+0435:Satzspiegel 30,5 x 24,5 cm
+0440.001:[S.l.]
+0720.001:Politische Flugblatt; Antiunionistisches Flugblatt
+0721.001:Dreissigjähriger Krieg; Böhmisch-Pfälzischer Krieg
+0723.001:Mu.hammad; Mohammed
+1105.001:Einbl. V,8 b-11
+1125.001:Blattmaß 32,5 x 26 cm; Hinterklebt
+1145.001:ei
+2005:Metallschnittleisten als Spaltentrenner; Textinitiale
+2006:Paas, John Roger: The German Political Broadsheet (1600 - 1700)
+2008:Mohammed; Zank; Hader; Katze
+2018:Versform; 4 Spalten
+2023:¤ 34B12 - Katze ¤ 25F(+51) - kämpfende Tiere; aggressive Beziehungen
+2029:002
+2030.001:http://zoom.bib-bvb.de/StyleServer/calcrgn?cat=einbl&item=/300000184_0_r.sid&wid=750&hei=500&style=bsb/einbl.xsl&plugin=false
+9999:
+0000:5
+0001:000000005
+0002:03.05.2001
+0003:29.08.2002
+0015:ger
+0036:m
+0331:Alles hatt Sein Zeydt
+0331.001:-- Hää?
+0424:1621
+0425:[1621]
+0433:1 Bl.
+0434:1 Ill.; 16 x 27,5 cm
+0435:Satspiegel 16 x 27,5 cm
+0440.001:[S.l.]
+0511:Erscheinungsjahr ermittelt aus Bildinhalt und Nachweis
+0511:Erscheinungsjahr ermittelt aus Bildinhalt und Nachweis
+0527.001:Andere Ausgabe: Einbl. V,8 b-8; Deutsche illustrierte Flugblätter des 16. und 17. Jahrhunderts
+2028:BLUB
+2029:003
+2030.001:http://zoom.bib-bvb.de/StyleServer/calcrgn?cat=einbl&item=/300000185_0_r.sid&wid=750&hei=500&style=bsb/einbl.xsl&plugin=false
+9999:

data/lib/athena/formats/dbm.rb

@@ -0,0 +1,66 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of athena, the database file converter.                         #
+#                                                                             #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# athena is free software; you can redistribute it and/or modify it under the #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# athena is distributed in the hope that it will be useful, but WITHOUT ANY   #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with athena. If not, see <http://www.gnu.org/licenses/>.                    #
+#                                                                             #
+###############################################################################
+#++
+
+require 'iconv'
+
+class Athena::Formats
+
+  class DBM < Athena::Formats
+
+    register_formats :out, 'dbm', 'midos'
+
+    CRLF = "\015\012"
+
+    ICONV_TO_LATIN1 = Iconv.new('latin1//TRANSLIT//IGNORE', 'utf-8')
+
+    VALUE_SEPARATOR  = '|'
+    RECORD_SEPARATOR = '&&&'
+
+    def self.convert(record)
+      dbm = ["ID:#{record.id}"]
+
+      record.struct.each { |field, struct|
+        strings = struct[:elements].inject([]) { |array, element|
+          values = (struct[:values][element] || []).map { |v|
+            (v || '').strip.gsub(/(?:\r?\n)+/, ' ')
+          }.reject { |v| v.empty? }
+
+          array << (values.empty? ? struct[:empty] : values.join(VALUE_SEPARATOR))
+        }
+
+        dbm << "#{field.to_s.upcase}:#{ICONV_TO_LATIN1.iconv(struct[:string] % strings)}"
+      }
+
+      dbm << RECORD_SEPARATOR
+
+      dbm.join(CRLF) << CRLF << CRLF
+    end
+
+  end
+
+end

data/lib/athena/formats/ferret.rb

@@ -0,0 +1,106 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of athena, the database file converter.                         #
+#                                                                             #
+# Copyright (C) 2007-2008 University of Cologne,                              #
+#                         Albertus-Magnus-Platz,                              #
+#                         50932 Cologne, Germany                              #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <jens.wille@uni-koeln.de>                                    #
+#                                                                             #
+# athena is free software; you can redistribute it and/or modify it under the #
+# terms of the GNU General Public License as published by the Free Software   #
+# Foundation; either version 3 of the License, or (at your option) any later  #
+# version.                                                                    #
+#                                                                             #
+# athena is distributed in the hope that it will be useful, but WITHOUT ANY   #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with athena. If not, see <http://www.gnu.org/licenses/>.                    #
+#                                                                             #
+###############################################################################
+#++
+
+require 'rubygems'
+
+gem 'ferret', ENV['FERRET_VERSION'] if ENV['FERRET_VERSION']
+require 'ferret'
+
+class Athena::Formats
+
+  class Ferret < Athena::Formats
+
+    register_format :in, 'ferret'
+
+    attr_reader :record_element, :config, :parser, :match_all_query
+
+    def initialize(parser)
+      config = parser.config.dup
+
+      case @record_element = config.delete(:__record_element)
+        when String
+          # fine!
+        when nil
+          raise NoRecordElementError, 'no record element specified'
+        else
+          raise IllegalRecordElementError, "illegal record element #{@record_element}"
+      end
+
+      @config = config
+      @parser = parser
+    end
+
+    def parse(source)
+      path = source.path
+
+      # make sure the index can be opened
+      begin
+        File.open(File.join(path, 'segments')) {}
+      rescue Errno::ENOENT, Errno::EACCES => err
+        raise "can't open index at #{path} (#{err.to_s.sub(/ - .*/, '')})"
+      end
+
+      index = ::Ferret::Index::IndexReader.new(path)
+      first, last = 0, index.max_doc - 1
+
+      # make sure we can read from the index
+      begin
+        index[first]
+        index[last]
+      rescue StandardError  # EOFError, "Not available", ...
+        raise "possible Ferret version mismatch; try to set the " <<
+              "FERRET_VERSION environment variable to something " <<
+              "other than #{Ferret::VERSION}"
+      end
+
+      first.upto(last) { |i|
+        unless index.deleted?(i)
+          doc = index[i]
+
+          Athena::Record.new(parser.block, doc[record_element]) { |record|
+            config.each { |element, field_config|
+              record.update(element, doc[element], field_config)
+            }
+          }
+        end
+      }
+
+      index.num_docs
+    end
+
+    private
+
+    class NoRecordElementError < StandardError
+    end
+
+    class IllegalRecordElementError < StandardError
+    end
+
+  end
+
+end