mabmapper 1.0.0.pre18 → 2.0.0

data/lib/mabmapper/cli.rb

@@ -3,6 +3,7 @@
 #
 require 'mabmapper/elasticsearch_writer'
 require 'mabmapper/tar_writer'
+require 'metacrunch/ubpb/transformations/mab_to_primo'
 
 module Mabmapper
   class Cli
@@ -12,7 +13,7 @@ module Mabmapper
     def initialize
       @options = {}
       parse_command_line!
-      load_engine!
+      @transformation = Metacrunch::UBPB::Transformations::MabToPrimo.new
       process_files!
     end
 
@@ -35,11 +36,6 @@ module Mabmapper
           @options[:debug] = true
         end
 
-        @options[:debug_fields] = []
-        opts.on( '-f', '--debug-fields a,b,c', Array, "If debug mode is on only fields matching the given names will be debugged." ) do |fields|
-          @options[:debug_fields] = fields
-        end
-
         @options[:silent] = false
         opts.on( '-s', '--silent', "Do not output anything on the console" ) do
           @options[:silent] = true
@@ -74,22 +70,6 @@ module Mabmapper
       (puts optparse.help ; exit)
     end
 
-    #
-    # Load normalization engine
-    #
-    def load_engine!
-      begin
-        engine_file = "mabmapper/aleph_mab_xml_engine" # TODO: Make me configurable
-        require engine_file
-        engine_class_name = "#{engine_file}".classify
-        @engine = engine_class_name.constantize.new
-        log "#{engine_class_name} loaded!"
-      rescue LoadError
-        log "Error loading engine #{engine_file}."
-        exit 1
-      end
-    end
-
     #
     # Process the input files
     #
@@ -119,6 +99,28 @@ module Mabmapper
 
   private
 
+    def hash_to_xml(hash)
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.document do
+          hash.each_pair do |_key, _values|
+            if _values.present? || _values == false
+              if _values.is_a?(Array)
+                #xml.send("#{field.name.downcase.pluralize}_") do
+                  _values.each do |_value|
+                    xml.send("#{_key.downcase}_", _value)
+                  end
+                #end
+              else
+                xml.send("#{_key.downcase}_", _values)
+              end
+            end
+          end
+        end
+      end
+
+      builder.to_xml
+    end
+
     def process_file(file)
       case
       when file.end_with?('.tar')    then process_tar_file(file)
@@ -137,13 +139,11 @@ module Mabmapper
       tarReader.each do |entry|
         if entry.file?
           log "Processing file #{entry.full_name} from archive #{file}"
-          result = @engine.process(entry.full_name, entry.read, archive: file)
+          result = @transformation.call(entry.read.force_encoding("utf-8"))
 
           writer.add_file(entry.full_name, 0644) do |f|
-            f.write(result.to_xml)
+            f.write(hash_to_xml(result))
           end if writer
-
-          log "Result for #{entry.full_name} from archive #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
         end
       end
 
@@ -163,14 +163,12 @@ module Mabmapper
         tarReader.each do |entry|
           if entry.file?
             log "Processing file #{entry.full_name} from archive #{file}"
-            result = @engine.process(entry.full_name, entry.read, archive: file)
+            result = @transformation.call(entry.read.force_encoding("utf-8"))
 
-            xml_result = result.to_xml
+            xml_result = hash_to_xml(result)
             writer.add_file_simple(entry.full_name, 0644, xml_result.bytesize) do |f|
               f.write(xml_result)
             end if writer
-
-            log "Result for #{entry.full_name} from archive #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
           end
         end
       ensure
@@ -181,14 +179,12 @@ module Mabmapper
 
     def process_default_file(file)
       log "Processing file #{file}"
-      result = @engine.process(file, File.open(file, "r").read)
+      result = @transformation.call(File.open(file, "r").read.force_encoding("utf-8"))
 
       if output_dir
         out_file = File.join(output_dir, File.basename(file))
-        File.open(out_file, 'w') { |f| f.write(result.to_xml) }
+        File.open(out_file, 'w') { |f| f.write(hash_to_xml(result)) }
       end
-
-      log "Result for #{file}\n#{result.to_xml(@options[:debug_fields])}\n" if @options[:debug]
     end
 
     def output_dir
@@ -210,6 +206,5 @@ module Mabmapper
         end
       end
     end
-
   end
 end

data/lib/mabmapper/engine.rb

@@ -84,7 +84,7 @@ module Mabmapper
         @engine = eval("self", block.binding)
       end
 
-      attr_reader :name, :result, :doc
+      attr_reader :name, :result, :doc, :source
 
       def ref(field_name)
         field = @engine.fields.find{ |f| f.name == field_name.to_s }
@@ -107,6 +107,7 @@ module Mabmapper
 
       def process(document)
         @doc    = document
+        @source = document.source
         @result = instance_eval(&@proc)
       end
     end

data/lib/mabmapper/mab_xml/document.rb

@@ -1,4 +1,5 @@
 # coding: utf-8
+require 'metacrunch/mab2'
 require 'stringex'
 
 module Mabmapper
@@ -6,11 +7,12 @@ module Mabmapper
     class Document
       include QueryHelper
 
-      attr_accessor :xml
+      attr_accessor :source, :xml
 
       def initialize(contents)
-        @xml = Nokogiri::XML(contents)
-        @xml.remove_namespaces!
+        @source = Metacrunch::Mab2::Document.from_aleph_mab_xml(contents)
+        #@xml = Nokogiri::XML(contents)
+        #@xml.remove_namespaces!
       end
 
       #

data/lib/mabmapper/version.rb

@@ -1,3 +1,3 @@
 module Mabmapper
-  VERSION = "1.0.0.pre18"
+  VERSION = "2.0.0"
 end

data/lib/mabmapper.rb

@@ -1,6 +1,7 @@
+require 'active_support'
+require 'active_support/core_ext'
 require 'rubygems/package'
 require 'optparse'
-require 'active_support/core_ext'
 require 'nokogiri'
 
 begin

data/mabmapper.gemspec

@@ -2,7 +2,7 @@
 require File.expand_path('../lib/mabmapper/version', __FILE__)
 
 Gem::Specification.new do |gem|
-  gem.authors       = ["René Sprotte"]
+  gem.authors       = ["René Sprotte", "Michael Sievers"]
   gem.email         = ["r.sprotte@ub.uni-paderborn.de"]
   gem.description   = %q{Mabmapper is a powerful and extendable processing engine to normalize any kind of
     input data into a simple intermediate format made of fields and values. It comes with a ready to use
@@ -21,14 +21,10 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_dependency('nokogiri', '~> 1.6.0')
-  gem.add_dependency('activesupport', '>= 4.0.0')
-  gem.add_dependency('libxml-ruby', '~> 2.7.0')
-  gem.add_dependency('oj', '~> 2.1.4')
-  gem.add_dependency('stringex', '~> 2.1.0')
-
-  gem.add_development_dependency('minitest',         '~> 4.7.5')
-  gem.add_development_dependency('pry',              '0.9.12.2') # stuck to 0.9.12.2 due to repl color issue
-  gem.add_development_dependency('pry-nav',          '~> 0.2.3')
-  gem.add_development_dependency('pry-syntax-hacks', '~> 0.0.6')
+  gem.add_dependency('nokogiri', '~> 1.6')
+  gem.add_dependency('activesupport', '>= 4.0')
+  gem.add_dependency('libxml-ruby', '~> 2.7')
+  gem.add_dependency('metacrunch-mab2')
+  gem.add_dependency('oj', '~> 2.1')
+  gem.add_dependency('stringex', '~> 2.1')
 end

data/spec/mabmapper/aleph_mab_xml_engine/publisher_spec.rb

@@ -0,0 +1,197 @@
+fieldname = "publisher"
+
+describe "field :#{fieldname}" do
+  # RAK
+  #
+  # f410 [NW]
+  #    a Ort des 1. Verlagers [WDH]
+  #
+  # f412 [NW]
+  #    a Name des 1. Verlegers [WDH]
+  #
+  # f415 [NW]
+  #    a Ort des 2. Verlegers
+  #
+  # f417 [NW]
+  #    a Name des 2. Verlegers
+  #
+  # RDA
+  #
+  # f419 [WDH]
+  #    a Ort(e) (ggf. mehrere durch ; getrennt)
+  #    b Verlagsname
+  #    c Datumsangabe
+  
+  context "one publisher, one place" do
+    context "for a RAK record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("410", ind2: "1") { subfield("a", "First publisher place") }
+          datafield("412", ind2: "1") { subfield("a", "First publisher") }
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher"]) }
+    end
+
+    context "for a RDA record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("419") do
+            subfield("a", "First publisher place")
+            subfield("b", "First publisher")
+          end
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher"]) }
+    end
+  end
+
+  context "one publisher, multiple places" do
+    context "for a RAK record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("410", ind2: "1") { subfield("a", "First publisher place") }
+          datafield("410", ind2: "1") { subfield("a", "Another place") }
+          datafield("412", ind2: "1") { subfield("a", "First publisher") }
+
+          datafield("415", ind2: "1") { subfield("a", "Second publisher place") }
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher"]) }
+    end
+
+    context "for a RDA record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("419") do
+            subfield("a", "First publisher place")
+            subfield("a", "First publisher second place")
+            subfield("b", "First publisher")
+          end
+
+          datafield("419") do
+            subfield("a", "Another place")
+          end
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher"]) }
+    end
+  end
+
+  context "multiple publishers, multiple places" do
+    context "for a RAK record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("410", ind2: "1") { subfield("a", "First publisher place") }
+          datafield("412", ind2: "1") { subfield("a", "First publisher") }
+
+          datafield("415", ind2: "1") { subfield("a", "Second publisher place") }
+          datafield("417", ind2: "1") { subfield("a", "Second publisher") }
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher", "Second publisher place : Second publisher"]) }
+    end
+
+    context "for a RDA record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("419") do
+            subfield("a", "First publisher place")
+            subfield("b", "First publisher")
+          end
+
+          datafield("419") do
+            subfield("a", "Second publisher place")
+            subfield("b", "Second publisher")
+          end
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher", "Second publisher place : Second publisher"]) }
+    end
+  end
+
+  context "multiple publishers, not all having an associated place" do
+    context "for a RAK record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("410", ind2: "1") { subfield("a", "First publisher place") }
+          datafield("412", ind2: "1") { subfield("a", "First publisher") }
+
+          datafield("417", ind2: "1") { subfield("a", "Second publisher") }
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher", "Second publisher"]) }
+    end
+
+    context "for a RDA record" do
+      subject do
+        mab_xml = mab_xml_builder do
+          datafield("419") do
+            subfield("a", "First publisher place")
+            subfield("b", "First publisher")
+          end
+
+          datafield("419") do
+            subfield("b", "Second publisher")
+          end
+        end
+
+        transform(mab_xml)[fieldname]
+      end
+
+      it { is_expected.to eq(["First publisher place : First publisher", "Second publisher"]) }
+    end
+  end
+
+  context "multiple publishers, multiple places within repeated subfields (RAK)" do
+    subject do
+      mab_xml = mab_xml_builder do
+        datafield("410", ind2: "1") { subfield("a", "First publisher place") }
+        datafield("410", ind2: "1") { subfield("a", "Second publisher place") }
+
+        datafield("412", ind2: "1") { subfield("a", "First publisher") }
+        datafield("412", ind2: "1") { subfield("a", "Second publisher") }
+      end
+
+      transform(mab_xml)[fieldname]
+    end
+
+    it { is_expected.to eq(["First publisher place : First publisher", "Second publisher place : Second publisher"]) }
+  end
+
+  context "multiple places for a single publisher (RDA)" do
+    subject do
+      mab_xml = mab_xml_builder do
+        datafield("419") do
+          subfield("a", "First publisher first place ; First publisher second place")
+          subfield("b", "First publisher")
+        end
+      end
+
+      transform(mab_xml)[fieldname]
+    end
+
+    it { is_expected.to eq(["First publisher first place : First publisher"]) }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,86 @@
+if ENV["CODECLIMATE_REPO_TOKEN"]
+  # report coverage only for latest mri ruby
+  if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.2.0"
+    require "codeclimate-test-reporter"
+    CodeClimate::TestReporter.start
+  end
+else
+  require "simplecov"
+  SimpleCov.start
+end
+
+require "mabmapper"
+require "mabmapper/aleph_mab_xml_engine"
+require "hashdiff"
+require "nokogiri"
+require "yaml"
+
+begin
+  require "pry"
+rescue LoadError
+end
+
+RSpec.configure do |config|
+  # begin --- rspec 3.1 generator
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+  # end --- rspec 3.1 generator
+end
+
+def asset_dir
+  File.expand_path(File.join(File.dirname(__FILE__), "assets"))
+end
+
+def mab_xml_builder(identifier="aleph-publish:000000000", &block)
+  Nokogiri::XML::Builder.new do
+    send(
+      :"OAI-PMH",
+      "xmlns" => "http://www.openarchives.org/OAI/2.0/",
+      "xmlns:xsi" => "http://www.w3.org/2001/XMLSchema-instance",
+      "xsi:schemaLocation" => "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+    ) do
+      ListRecords do
+        record do
+          header do |_xml|
+            _xml.identifier identifier
+          end
+          metadata do
+            record("xmlns" => "http://www.ddb.de/professionell/mabxml/mabxml-1.xsd") do
+              define_singleton_method(:controlfield) do |_tag, _text|
+                send(:method_missing, :controlfield, _text, tag: _tag)
+              end
+
+              define_singleton_method(:datafield) do |_tag, _attributes = {}, &_block|
+                send(:method_missing, :datafield, {tag: _tag}.merge(_attributes), &_block)
+              end
+
+              define_singleton_method(:subfield) do |_code, _text|
+                send(:method_missing, :subfield, _text, code: _code)
+              end
+
+              define_singleton_method(:journal!) do
+                controlfield("052", "p")
+              end
+
+              instance_eval(&block)
+            end
+          end
+        end
+      end
+    end
+  end
+  .to_xml
+end
+
+def transform(mab_xml)
+  Mabmapper::AlephMabXmlEngine.new.process(nil, mab_xml).to_hash
+end
+
+def read_asset(path_to_file)
+  File.read(File.expand_path(File.join(asset_dir, path_to_file)))
+end

data/test/mab_files/test_description/405.xml

@@ -8,12 +8,12 @@
       <metadata>
         <record xmlns="http://www.ddb.de/professionell/mabxml/mabxml-1.xsd">
           <datafield tag="405" ind2="1">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">YYY</subfield>
+            <subfield code="a">YYY</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="405" ind2="2">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
         </record>
       </metadata>

data/test/mab_files/test_description/501-519.xml

@@ -8,20 +8,20 @@
       <metadata>
         <record xmlns="http://www.ddb.de/professionell/mabxml/mabxml-1.xsd">
           <datafield tag="501" ind2="1">
-            <subfield code="a">AAA</subfield>
-            <subfield code="p">BBB</subfield>
+            <subfield code="a">BBB</subfield>
+            <subfield code="p">AAA</subfield>
           </datafield>
           <datafield tag="501" ind2="2">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="519" ind2="1">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">YYY</subfield>
+            <subfield code="a">YYY</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="519" ind2="2">
-            <subfield code="a">YYY</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">YYY</subfield>
           </datafield>
         </record>
       </metadata>

data/test/mab_files/test_description/522.xml

@@ -8,12 +8,12 @@
       <metadata>
         <record xmlns="http://www.ddb.de/professionell/mabxml/mabxml-1.xsd">
           <datafield tag="522" ind2="1">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">YYY</subfield>
+            <subfield code="a">YYY</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="522" ind2="2">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
         </record>
       </metadata>

data/test/mab_files/test_description/523.xml

@@ -8,12 +8,12 @@
       <metadata>
         <record xmlns="http://www.ddb.de/professionell/mabxml/mabxml-1.xsd">
           <datafield tag="523" ind2="1">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">YYY</subfield>
+            <subfield code="a">YYY</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="523" ind2="2">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
         </record>
       </metadata>

data/test/mab_files/test_description/536-537.xml

@@ -8,20 +8,20 @@
       <metadata>
         <record xmlns="http://www.ddb.de/professionell/mabxml/mabxml-1.xsd">
           <datafield tag="536" ind2="1">
-            <subfield code="a">AAA</subfield>
-            <subfield code="p">BBB</subfield>
+            <subfield code="a">BBB</subfield>
+            <subfield code="p">AAA</subfield>
           </datafield>
           <datafield tag="536" ind2="2">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="537" ind2="1">
-            <subfield code="a">XXX</subfield>
-            <subfield code="p">YYY</subfield>
+            <subfield code="a">YYY</subfield>
+            <subfield code="p">XXX</subfield>
           </datafield>
           <datafield tag="537" ind2="2">
-            <subfield code="a">YYY</subfield>
-            <subfield code="p">ZZZ</subfield>
+            <subfield code="a">ZZZ</subfield>
+            <subfield code="p">YYY</subfield>
           </datafield>
         </record>
       </metadata>