mabmapper 1.0.0.pre15

data/lib/mabmapper/cli.rb

@@ -0,0 +1,216 @@
+#
+# The command line interface class
+#
+require 'rubygems/test_utilities'
+require 'mabmapper/elasticsearch_writer'
+require 'mabmapper/tar_writer'
+
+module Mabmapper
+  class Cli
+
+    ROOT_DIR = Dir.pwd
+
+    def initialize
+      @options = {}
+      parse_command_line!
+      load_engine!
+      process_files!
+    end
+
+  protected
+
+    #
+    # Parse command line options
+    #
+    def parse_command_line!
+      optparse = OptionParser.new do |opts|
+        opts.banner = "Usage: mabmapper [options] FILES"
+
+        @options[:output_dir] = nil
+        opts.on( '-o', '--output DIR', 'Output directory' ) do |dir|
+          @options[:output_dir] = dir
+        end
+
+        @options[:debug] = false
+        opts.on( '-d', '--debug', "Debug mode on." ) do
+          @options[:debug] = true
+        end
+
+        @options[:debug_fields] = []
+        opts.on( '-f', '--debug-fields a,b,c', Array, "If debug mode is on only fields matching the given names will be debugged." ) do |fields|
+          @options[:debug_fields] = fields
+        end
+
+        @options[:silent] = false
+        opts.on( '-s', '--silent', "Do not output anything on the console" ) do
+          @options[:silent] = true
+        end
+
+        #@options[:engine] = "mabmapper/aleph_mab_xml"
+        #opts.on( '-e', '--engine ENGINE', 'Normalization engine (Default: aleph_mab_xml)' ) do |engine|
+        #  @options[:engine] = engine
+        #end
+
+        @options[:no_of_procs] = 1
+        opts.on( '-n', '--number-of-procs NUM', Integer, "Use NUM parallel procs [Default: 1]" ) do |n|
+          @options[:no_of_procs] = n
+        end
+
+        @options[:writer] = Mabmapper::TarWriter
+        opts.on( '-w', '--writer WRITER', "Use specified writer (elasticsearch|tar) [Default: tar]" ) do |writer|
+          @options[:writer] = Mabmapper::ElasticSearchWriter if writer.downcase == 'elasticsearch'
+        end
+
+        opts.on( '-h', '--help', 'Display this screen' ) do
+         puts opts
+         exit
+        end
+      end
+      optparse.parse!
+
+      # Check for required file arguments
+      (puts optparse.help ; exit) unless ARGV.present?
+    rescue OptionParser::ParseError => e
+      puts e.message
+      (puts optparse.help ; exit)
+    end
+
+    #
+    # Load normalization engine
+    #
+    def load_engine!
+      begin
+        engine_file = "mabmapper/aleph_mab_xml_engine" # TODO: Make me configurable
+        require engine_file
+        engine_class_name = "#{engine_file}".classify
+        @engine = engine_class_name.constantize.new
+        log "#{engine_class_name} loaded!"
+      rescue LoadError
+        log "Error loading engine #{engine_file}."
+        exit 1
+      end
+    end
+
+    #
+    # Process the input files
+    #
+    def process_files!
+      max_processes = @options[:no_of_procs]
+
+      # Prepare joblists for each process
+      joblist = max_processes.times.map do |i|
+        step = (ARGV.size/max_processes.to_f).ceil
+        ARGV.slice(i * step, step)
+      end.compact
+
+      # Run joblist
+      joblist.each_with_index do |list, index|
+        fork do
+          Thread.current[:name] = "Process #{index}"
+          list.each do |file|
+            process_file(file)
+          end
+        end
+      end
+
+      Process.waitall
+
+      log "FINISHED"
+    end
+
+  private
+
+    def process_file(file)
+      case
+      when file.end_with?('.tar')    then process_tar_file(file)
+      when file.end_with?('.tar.gz') then process_tar_gz_file(file)
+      else process_default_file(file)
+      end
+    end
+
+    def process_tar_file(file)
+      writer = if output_dir
+        out_file = @options[:writer].out_file(output_dir, File.basename(file))
+        @options[:writer].new(File.open(out_file, 'w'))
+      end
+
+      tarReader = Gem::Package::TarReader.new(File.open(file, 'r'))
+      tarReader.each do |entry|
+        if entry.file?
+          log "Processing file #{entry.full_name} from archive #{file}"
+          result = @engine.process(entry.full_name, entry.read, archive: file)
+
+          writer.add_file(entry.full_name, 0644) do |f|
+            f.write(result.to_xml)
+          end if writer
+
+          log "Result for #{entry.full_name} from archive #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
+        end
+      end
+
+      writer.close if writer
+    end
+
+    def process_tar_gz_file(file)
+      if output_dir
+        out_file   = @options[:writer].out_file(output_dir, File.basename(file), will_be_gziped: true)
+        fileIO     = File.open(out_file, "w")
+        gzipWriter = Zlib::GzipWriter.new(fileIO)
+        writer = @options[:writer].new(gzipWriter)
+      end
+
+      begin
+        tarReader = Gem::Package::TarReader.new(Zlib::GzipReader.open(file))
+        tarReader.each do |entry|
+          if entry.file?
+            log "Processing file #{entry.full_name} from archive #{file}"
+            result = @engine.process(entry.full_name, entry.read, archive: file)
+
+            xml_result = result.to_xml
+            writer.add_file_simple(entry.full_name, 0644, xml_result.bytesize) do |f|
+              f.write(xml_result)
+            end if writer
+
+            log "Result for #{entry.full_name} from archive #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
+          end
+        end
+      ensure
+        writer.close     if writer
+        gzipWriter.close if gzipWriter && !gzipWriter.closed?
+      end
+    end
+
+    def process_default_file(file)
+      log "Processing file #{file}"
+      result = @engine.process(file, File.open(file, "r").read)
+
+      if output_dir
+        out_file = File.join(output_dir, File.basename(file))
+        File.open(out_file, 'w') { |f| f.write(result.to_xml) }
+      end
+
+      log "Result for #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
+    end
+
+    def output_dir
+      if @options[:output_dir]
+        dir = File.expand_path(@options[:output_dir])
+        raise "No such dir #{dir}" unless Dir.exists?(dir)
+        return dir
+      end
+    end
+
+    def log(message)
+      unless @options[:silent]
+        current_thread_name      = Thread.current[:name]
+        current_thread_indicator = "[#{current_thread_name}]"
+        if current_thread_name.present?
+          puts "#{current_thread_indicator} #{message}"
+        else
+          puts "#{message}"
+        end
+      end
+    end
+
+  end
+end

data/lib/mabmapper/elasticsearch_writer.rb

@@ -0,0 +1,52 @@
+require 'active_support/core_ext'
+require 'libxml'
+require 'oj'
+
+#
+# (compressed) bulk files can be posted using curl, e.g. with the following command
+#
+# gzip -c -d aleph.PRIMO.20120908.091506.1.es_bulk.gz | curl -XPOST 'localhost:9200/catalog/record/_bulk' --data-binary @-
+#
+module Mabmapper
+  class ElasticSearchWriter
+
+    def self.out_file(output_dir_name, file_name, options = {})
+      file_basename = File.basename(file_name).gsub(/\.tar.gz|\.tar|\.tgz/, '.es_bulk')
+      file_basename << '.gz' if options[:will_be_gziped] === true
+
+      File.join(output_dir_name, file_basename)
+    end
+
+    def initialize(io)
+      @io = io
+
+      # Set libxml as minixml backend to improve performance
+      ActiveSupport::XmlMini.backend = 'LibXML'
+    end
+
+    def add_file(name, mode) # :yields: io
+      yield self
+    end
+
+    def add_file_simple(name, mode, size) # :yields: io
+      yield self
+    end
+
+    def close
+      @io.close unless @io.closed?
+    end
+
+    # we assume that data is string serialized xml
+    def write(xml)
+      bulk = []
+      hash = Hash.from_xml(xml)
+
+      bulk.push(Oj.dump({ index: { _id: "#{hash['document']['id']}" }}, mode: :compat))
+      bulk.push(Oj.dump(hash['document'], mode: :compat))
+
+      # Beware, right positions of newlines is vital for elasticsearch bulk import
+      @io.write(bulk.join("\n") << "\n")
+    end
+
+  end
+end

data/lib/mabmapper/engine.rb

@@ -0,0 +1,112 @@
+# coding: utf-8
+module Mabmapper
+  class Engine
+
+    def process(input_filename, contents, options={archive: nil})
+      document_class = self.class.document_class_value
+      raise "The engine does't provide a document class. This is required." unless document_class.present?
+      document = document_class.new(contents)
+      fields.each do |field|
+        begin
+          field.send(:process, document)
+        rescue => e
+          puts "----------------------------"
+          puts "Error  : #{e.message}"
+          puts "Field  : #{field.name}"
+          puts "File   : #{input_filename}"
+          puts "Archive: #{options[:archive]}" if options[:archive].present?
+          puts "----------------------------"
+          puts "Backtrace:"
+          puts e.backtrace
+          puts "----------------------------"
+          puts
+        end
+      end
+      self
+    end
+
+    def fields
+      self.class.fields
+    end
+
+    def to_xml(only_fields = [])
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.document do
+          fields.each do |field|
+            if field.result.present? || field.result == false
+              next if only_fields.present? and not only_fields.include?(field.name)
+              if field.result.is_a?(Array)
+                #xml.send("#{field.name.downcase.pluralize}_") do
+                  field.result.each do |result|
+                    xml.send("#{field.name.downcase}_", result)
+                  end
+                #end
+              else
+                xml.send("#{field.name.downcase}_", field.result)
+              end
+            end
+          end
+        end
+      end
+
+      builder.to_xml
+    end
+
+    def to_hash
+      hash = {}
+      fields.each { |field| hash[field.name.downcase] = field.result }
+      hash
+    end
+
+  protected
+
+    class << self
+      cattr_accessor :document_class_value
+      cattr_accessor :fields
+
+      def document_class(value)
+        self.document_class_value = value
+      end
+
+      def field(name, &block)
+        self.fields ||= []
+        self.fields << Field.new(name, &block) if block_given?
+      end
+    end
+
+    # @see: http://www.dan-manges.com/blog/ruby-dsls-instance-eval-with-delegation
+    class Field
+      def initialize(name, &block)
+        @name   = name.to_s
+        @proc   = block
+        @engine = eval("self", block.binding)
+      end
+
+      attr_reader :name, :result, :doc
+
+      def ref(field_name)
+        field = @engine.fields.find{ |f| f.name == field_name.to_s }
+        raise "No such field #{field_name} in section #{section_name}" if field.blank?
+        field.result
+      end
+
+      def merge(value1, value2, options = {delimiter: ' ', wrap: nil})
+        v1 = [value1].map(&:presence).compact.join(options[:delimiter])
+        v2 = [value2].map(&:presence).compact.join(options[:delimiter])
+        v2 = wrap(v2, options[:wrap]) if v2.present? and options[:wrap].present?
+        [v1, v2].map(&:presence).compact.join(options[:delimiter])
+      end
+
+      def wrap(value, pattern)
+        pattern.gsub("@", value)
+      end
+
+    private
+
+      def process(document)
+        @doc    = document
+        @result = instance_eval(&@proc)
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/document.rb

@@ -0,0 +1,53 @@
+# coding: utf-8
+require 'stringex'
+
+module Mabmapper
+  module MabXml
+    class Document
+      include QueryHelper
+
+      def initialize(contents)
+        @xml = Nokogiri::XML(contents)
+        @xml.remove_namespaces!
+      end
+
+      #
+      # Returns the contents of a MAB control field as an array.
+      #
+      def controlfield(name)
+        xpath  = "//controlfield[@tag='#{name}']"
+        result = @xml.at_xpath(xpath).try(:text)
+        result.present? ? result.chars.to_a.map{|e| (e=='|') ? nil : e} : []
+      end
+
+      # Normalize text based on the NACO rules
+      def naco_normalization(value)
+        return unless value.present?
+
+        # Downcase everything
+        value.downcase!
+
+        # Convert unicode [and accented ASCII] characters to their
+        # plain-text ASCII equivalents
+        Stringex::Localization.backend = :internal
+        Stringex::Localization.locale = :de
+        Stringex::Localization.store_translations(:de, :transliterations, {"ü" => "ue", "ä" => "ae", "ö" => "oe", "ß" => "ss"}) if Stringex::Localization.backend.translations.blank?
+        value = value.to_ascii
+
+        # Convert special chars to spaces
+        value.gsub!(/[,$~^%*\/?@.:;<>{}!\(\)\-]/, ' ')
+
+        # Convert special chars to spaces
+        value.gsub!(/[\[\]\|]/, '')
+
+        # Remove leading and trailing spaces
+        value.strip!
+
+        # Condense multiple spaces
+        value.gsub!(/\s+/, ' ')
+
+        value
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/field.rb

@@ -0,0 +1,43 @@
+module Mabmapper
+  module MabXml
+    class Field
+      attr_reader :name, :subfields
+
+      def initialize(name)
+        @name      = name
+        @subfields = []
+      end
+
+      def add_subfield(subfield)
+        raise if subfield.blank? or not subfield.is_a?(Subfield)
+
+        @subfields << subfield
+      end
+
+      def values(options = {})
+        options.reverse_merge!(join_subfields: nil)
+        values = @subfields.map(&:value)
+
+        join = options[:join_subfields]
+        join ? values.join((!!join==join) ? nil : join.to_s) : values
+      end
+
+      def value(options = {})
+        options.reverse_merge!(join_subfields: ' ')
+        values(options)
+      end
+
+      def has_subfield?(subfield)
+        subfields.any?{|s| s.name.to_s == subfield.to_s}
+      end
+
+      def get_subfield(subfield)
+        subfields.select{|s| s.name.to_s == subfield.to_s}.try(:first)
+      end
+
+      def to_s
+        value
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/query.rb

@@ -0,0 +1,25 @@
+module Mabmapper
+  module MabXml
+    class Query
+      include QueryHelper
+
+      attr_reader :xml, :fields, :subfields
+
+      def initialize(xml)
+        @xml       = xml
+        @fields    = []
+        @subfields = []
+      end
+
+      def add_field(name, options={})
+        @fields << {name: name, options: options.reverse_merge(ind1: nil, ind2: nil)}
+        self
+      end
+
+      def add_subfield(name)
+        @subfields << {name: name}
+        self
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/query_helper.rb

@@ -0,0 +1,101 @@
+module Mabmapper
+  module MabXml
+    module QueryHelper
+
+      def field(name_or_enum, options={})
+        if name_or_enum.is_a?(String)
+          query.add_field(name_or_enum, options)
+        elsif name_or_enum.is_a?(Enumerable)
+          name_or_enum.inject(query) { |q, name| q.add_field(name, options) }
+        end
+      end
+
+      def subfield(name_or_enum)
+        if name_or_enum.is_a?(String)
+          query.add_subfield(name_or_enum)
+        elsif name_or_enum.is_a?(Enumerable)
+          name_or_enum.inject(query) { |q, name| q.add_subfield(name) }
+        end
+      end
+
+      def get
+        results = ResultSet.new
+
+        query.xml.xpath(fields_xpath).each do |field_xml|
+          name   = field_xml.attribute('tag').to_s
+          field  = Field.new(name)
+
+          field_xml.xpath(subfields_xpath).each do |subfield_xml|
+            name  = subfield_xml.attribute('code').to_s
+            value = subfield_xml.text.presence
+
+            subfield = Subfield.new(name, value)
+            field.add_subfield(subfield)
+          end
+
+          results.add_field(field)
+        end
+
+        results
+      end
+
+    private
+
+      def query
+        self.is_a?(Query) ? self : Query.new(@xml)
+      end
+
+      def fields_xpath
+        options = query.fields.map do |f|
+          name = f[:name]
+          ind1 = indicator_xpath("ind1", f[:options][:ind1])
+          ind2 = indicator_xpath("ind2", f[:options][:ind2])
+
+          s = []
+          s << "@tag='#{name}'"
+          s << ind1 if ind1
+          s << ind2 if ind2
+
+          "(#{s.join(' and ')})"
+        end.join(' or ')
+
+        "/OAI-PMH/ListRecords/record/metadata/record/datafield[#{options}]"
+      end
+
+      def subfields_xpath
+        global_negation = false
+
+        options = query.subfields.map do |f|
+          name = f[:name]
+
+          negation = name.starts_with?('-') and name.length > 1
+          name = negation ? name.slice(1..-1) : name
+          global_negation = true if negation
+
+          negation ? "not(@code='#{name}')" : "@code='#{name}'"
+        end
+
+        options = global_negation ? options.join(' and ') : options.join(' or ')
+
+        options.present? ? "subfield[#{options}]" : "subfield"
+      end
+
+      def indicator_xpath(name, value)
+        global_negation = false
+
+        options = [*value].map do |value|
+          if value
+            negation = value.starts_with?('-') and value.length > 1
+            global_negation = true if negation
+
+            negation ? "not(@#{name}='#{value.slice(1..-1)}')" : "@#{name}='#{value}'"
+          end
+        end
+
+        options = global_negation ? options.join(' and ') : options.join(' or ')
+
+        options.present? ? "(#{options})" : nil
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/result_set.rb

@@ -0,0 +1,34 @@
+module Mabmapper
+  module MabXml
+    class ResultSet
+      include Enumerable
+      attr_reader :fields
+
+      def initialize
+        @fields = []
+      end
+
+      def add_field(field)
+        raise if field.blank? or not field.is_a?(Field)
+
+        @fields << field
+      end
+
+      def each
+        @fields.each{|i| yield i}
+      end
+
+      def values(options = {})
+        @fields.map{|f| f.values(options)}
+      end
+
+      def value(options = {})
+        @fields.first.try(:value, options)
+      end
+
+      def to_s
+        value
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml/subfield.rb

@@ -0,0 +1,12 @@
+module Mabmapper
+  module MabXml
+    class Subfield
+      attr_reader :name, :value
+
+      def initialize(name, value)
+        @name  = name
+        @value = value
+      end
+    end
+  end
+end

data/lib/mabmapper/mab_xml.rb

@@ -0,0 +1,6 @@
+require 'mabmapper/mab_xml/query_helper'
+require 'mabmapper/mab_xml/query'
+require 'mabmapper/mab_xml/result_set'
+require 'mabmapper/mab_xml/field'
+require 'mabmapper/mab_xml/subfield'
+require 'mabmapper/mab_xml/document'

data/lib/mabmapper/tar_writer.rb

@@ -0,0 +1,29 @@
+module Mabmapper
+  class TarWriter
+
+    def self.out_file(output_dir_name, file_name, options = {})
+      File.join(output_dir_name, File.basename(file_name))
+    end
+
+    def initialize(io)
+      @tar_writer = Gem::Package::TarWriter.new(io)
+    end
+
+    def add_file(name, mode) # :yields: io
+      @tar_writer.add_file(name, mode) do |f|
+        yield f
+      end
+    end
+
+    def add_file_simple(name, mode, size) # :yields: io
+      @tar_writer.add_file_simple(name, mode, size) do |f|
+        yield f
+      end
+    end
+
+    def close
+      @tar_writer.close
+    end
+
+  end
+end

data/lib/mabmapper/version.rb

@@ -0,0 +1,3 @@
+module Mabmapper
+  VERSION = "1.0.0.pre15"
+end