mabmapper 1.0.0.pre15 → 1.0.0.pre16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edfa34f2b352b77bf53e7f581e3d60e8c59c84bf
-  data.tar.gz: 4dad71a39bddd155c2c3595a0995278a882e67a1
+  metadata.gz: c386816c140e51c20adc3f14f7a9e3366f5c0fe8
+  data.tar.gz: 1712293a68164eea826a58fe020ffb717d130411
 SHA512:
-  metadata.gz: af04565ff5c6bd6fa406a124b865fd412353e3bae166cb798054cf79ec66e9632416269753539a5f3d2201f0c9831e30c702f31416d481efc6bde4392c3ecb7b
-  data.tar.gz: 8445b5bc982ee1254c110cf30b93508703c11b4d1f2d4dc6868b061fc7fd6bf86dffb18144d667d041c35940c35f06d36dc41ce5e960dfa474816b67e7701661
+  metadata.gz: c8fbde45856cb171d4a762f477a019ae6a1958e7061ed0511f47ab08d977679d13d599f8166f049bdf2e53f51f13ac1bc02e69d0de32e84babab971429a317be
+  data.tar.gz: 761f5ed07daec64d18e6d53aac08f6b886026600cdca9762569b42933288d25d43f0fcbf03bb9bccbbcee509e521fcd0c2f2768e798be0b8f4f4fe8358be0102

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+Changelog
+=========
+
+* [2013-12-19] Added 334 to short_title_display
+* [2013-12-18] Take all aleph expanded 200er fields and their order into account for signature
+* [2013-11-29] Do not remove superorder_display entries without valid ht_number, just check for label existence
+* [2013-11-29] Added author_statement field (mab 359)
+* [2013-11-29] Added person_creator_display field
+* [2013-11-29] Added person_contributor_display field
+* [2013-11-29] Added corporate_body_creator_display field
+* [2013-11-29] Added corporate_body_contributor_display field
+* [2013-11-27] For short_title_display, take mab 310 into account
+* [2013-11-27] Fixed handling of ind1: ['-', ...] (the '-' caused trouble)

data/lib/mabmapper/aleph_mab_xml_engine.rb

@@ -24,6 +24,8 @@ module Mabmapper
       # Standort Detmold unterdrücken
       detmold_locations = doc.field('LOC').subfield('n').get.values.flatten
       value = 'D' if detmold_locations.present? && detmold_locations.all?{|v| v == '50'}
+      # Interimsaufnahmen unterdrücken
+      value = 'D' if doc.field('537', ind1: '-', ind2: '1').subfield('a').get.values.flatten.any? { |v| v.downcase.include? 'interimsaufnahme' }
 
       value
     end
@@ -161,14 +163,21 @@ module Mabmapper
     end
 
     field :short_title_display do
-      # 331 => Titel, 335 Titelzusatz (ind2 == 1 -> betrachteter Titel, ind2 == 2 -> Überordnung)
-      f089_1 = doc.field('089', ind2: '1').subfield('a').get.value
-      f331_1 = doc.field('331', ind2: '1').subfield('a').get.value
-      f331_2 = doc.field('331', ind2: '2').subfield('a').get.value
-      f335_1 = doc.field('335', ind2: '1').subfield('a').get.value
-
-      short_title = if (f331_1 && f335_1)
+      f089_1 = doc.field('089', ind2: '1').subfield('a').get.value                   # Bandangabe in Vorlageform 
+      f310_1 = doc.field('310', ind1: ['-', 'a'], ind2: '1').subfield('a').get.value # Hauptsachtitel in Ansetzungsform
+      f331_1 = doc.field('331', ind2: '1').subfield('a').get.value                   # Hauptsachtitel in Vorlageform oder Mischform
+      f331_2 = doc.field('331', ind2: '2').subfield('a').get.value                   #
+      f334_1 = doc.field('334', ind1: '-', ind2: '1').get.value                      # Allgemeine Materialbenennung 
+      f335_1 = doc.field('335', ind2: '1').subfield('a').get.value                   # Zusätze zum Hauptsachtitel
+
+      short_title = if f310_1
+        f310_1
+      elsif (f331_1 && f334_1 && f335_1)
+        "#{f331_1} [#{f334_1}] : #{f335_1}"
+      elsif (f331_1 && f335_1)
         "#{f331_1} : #{f335_1}"
+      elsif f331_1 && f334_1
+        "#{f331_1} [#{f334_1}]"
       elsif f331_1
         f331_1
       elsif f089_1 && f089_1.length > 3 && f089_1[/\A(\d|\s)+\Z/].nil? && !['buch', 'hauptbd.'].include?(f089_1.gsub(/\[|\]/, '').downcase)
@@ -209,13 +218,115 @@ module Mabmapper
       search_titles << doc.field('627', ind2: '1').subfield('a').get.values
       search_titles << doc.field('633', ind2: '1').subfield('a').get.values
 
-      search_titles.flatten.map(&:presence).compact.uniq
+      search_titles
+      .flatten
+      .compact
+      .map do |search_title|
+        # add entries without dashes in words, if words with dashes are present
+        if search_title.match(/[A-ZÄÖÜ][a-zäöü]+\-[A-ZÄÖÜ][a-zäöü]+/)
+          [search_title, search_title.split(' ').map { |string| string.gsub(/([A-ZÄÖÜ][a-zäöü]+)\-([A-ZÄÖÜ][a-zäöü]+)/, '\1\2').downcase.capitalize }.join(' ')]
+        else
+          search_title
+        end
+      end
+      .flatten
+      .map(&:presence).compact.uniq
+    end
+
+
+    #
+    # creator
+    #
+    field :person_creator_display do
+      creators = []
+
+      # Personen
+      (100..196).step(4) do |f|
+        doc.field("#{f}", ind1: ['-', 'a'], ind2: ['1', '2']).get.fields.each do |field|
+          creators << (field.get_subfield('a').try(:value) || field.get_subfield('p').try(:value) || field.get_subfield('c').try(:value))
+        end
+      end
+
+      creators.map(&:presence).compact.uniq
+    end
+
+    def self.corporate_body_from_field(field)
+      subfield_a = field.get_subfield('a').try(:value) # Körperschafts-/Kongressname/Geografikum ohne IDN-Verknüpfung (NW)
+      subfield_b = field.get_subfield('b').try(:value) # Unterordnung
+      subfield_c = field.get_subfield('c').try(:value) # Ort (NW)
+      subfield_d = field.get_subfield('d').try(:value) # Datum (NW)
+      subfield_e = field.get_subfield('e').try(:value) # Kongressname (NW)
+      subfield_g = field.get_subfield('g').try(:value) # Name des Geografikums (NW)
+      subfield_h = field.get_subfield('h').try(:value) # Zusatz
+      subfield_k = field.get_subfield('k').try(:value) # Körperschaftsname (NW)
+      subfield_n = field.get_subfield('n').try(:value) # Zählung (W)
+      subfield_x = field.get_subfield('x').try(:value) # nachgeordneter Teil (W)
+      subfield_z = field.get_subfield('x').try(:value) # geografische Unterteilung (W)
+
+      if !subfield_a && subfield_b && !subfield_c && !subfield_e && !subfield_g && subfield_h && subfield_k && !subfield_x && !subfield_z
+        "#{subfield_k} <#{subfield_h}> / #{subfield_b}"
+      else
+        [
+          subfield_a,
+          subfield_k,
+          subfield_e,
+          subfield_g,
+          subfield_b ? "/ #{subfield_b}" : nil,
+          "<#{[subfield_h, subfield_n, subfield_d, subfield_c, subfield_x, subfield_z].compact.join(', ').presence}>",
+        ].compact.join(' ').try(:sub, '<>', '').try(:strip)
+      end
+    end
+
+    field :corporate_body_creator_display do
+      creators = []
+
+      # Körpferschaften
+      (200..296).step(4) do |f|
+        doc.field("#{f}", ind1: ['-', 'a'], ind2: ['1', '2']).get.fields.each do |field|
+          creators << engine.corporate_body_from_field(field)
+        end
+      end
+
+      creators.map(&:presence).compact.uniq
     end
 
+    field :author_statement do
+      f359_1 = doc.field('359', ind1:"-", ind2:'1').subfield('a').get.values.flatten.presence
+      f359_2 = doc.field('359', ind1:"-", ind2:'2').subfield('a').get.values.flatten.presence
+      f359_1 || f359_2
+    end
 
     #
-    # Creator
+    # contributor
     #
+    field :person_contributor_display do
+      contributors = []
+
+      # Personen
+      (100..196).step(4) do |f|
+        doc.field("#{f}", ind1: ['b', 'c', 'e', 'f'], ind2: ['1', '2']).get.fields.each do |field|
+          name = (field.get_subfield('a').try(:value) || field.get_subfield('p').try(:value) || field.get_subfield('c').try(:value))
+          action_designator = field.get_subfield('b').try(:value)
+          contributors << [name, action_designator].map(&:presence).compact.join(' ')
+        end
+      end
+
+      contributors.map(&:presence).compact.uniq
+    end
+
+    field :corporate_body_contributor_display do
+      contributors = []
+
+      # Körpferschaften
+      (200..296).step(4) do |f|
+        doc.field("#{f}", ind1: ['b', 'c', 'e', 'f'], ind2: ['1', '2']).get.fields.each do |field|
+          contributors << engine.corporate_body_from_field(field)
+        end
+      end
+
+      contributors.map(&:presence).compact.uniq
+    end
+
     field :creator_contributor_display do
       creators = []
 
@@ -491,33 +602,6 @@ module Mabmapper
       ddc_fields.flatten.map(&:presence).compact.uniq
     end
 
-    #
-    # Description
-    #
-    field :description do
-      descriptions = []
-
-      # 405 - Erscheinungsverlauf von Zeitschriften
-      descriptions << doc.field('405', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
-
-      # 522 - Teilungsvermerk bei fortlaufenden Sammelwerken
-      descriptions << doc.field('522', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
-
-      # 523 - Erscheinungsverlauf von Monos
-      descriptions << doc.field('523', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
-
-      (501..519).each do |f|
-        descriptions << doc.field("#{f}", ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
-      end
-
-      (536..537).each do |f|
-        descriptions << doc.field("#{f}", ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
-      end
-
-      # Finally...
-      descriptions.flatten.map(&:presence).compact.uniq
-    end
-
     #
     # Abstracts
     #
@@ -553,6 +637,13 @@ module Mabmapper
         }
       end
 
+      if (f022a = doc.field('022').subfield('a').get.value).present?
+        relations << {
+          ht_number: f022a,
+          label: 'Sekundärform'
+        }
+      end
+
       (526..534).each do |mab_field_number|
         doc.field("#{mab_field_number}", ind2: '1').get.fields.each do |field|
           ht_number = field.get_subfield('9').try(:value).presence
@@ -592,7 +683,7 @@ module Mabmapper
 
       superorders
       .map(&:presence)
-      .delete_if { |element| element[:ht_number].blank? }
+      .delete_if { |element| element[:label].blank? }
       .each do |element|
         # remove 'not sort' indicators from label
         element[:label].try(:gsub!, /<<|>>/, '')
@@ -705,6 +796,33 @@ module Mabmapper
       type
     end
 
+    #
+    # Description
+    #
+    field :description do
+      descriptions = []
+
+      # 405 - Erscheinungsverlauf von Zeitschriften
+      descriptions << doc.field('405', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
+
+      # 522 - Teilungsvermerk bei fortlaufenden Sammelwerken
+      descriptions << doc.field('522', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
+
+      # 523 - Erscheinungsverlauf von Monos
+      descriptions << doc.field('523', ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
+
+      (501..519).each do |f|
+        descriptions << doc.field("#{f}", ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ')
+      end
+
+      (536..537).each do |f|
+        descriptions << doc.field("#{f}", ind2: '1').subfield(['a', 'p']).get.values(join_subfields: ': ') unless f == 537 && ref(:erscheinungsform) == "journal"
+      end
+
+      # Finally...
+      descriptions.flatten.map(&:presence).compact.uniq
+    end
+
     #
     # Delivery Catagory
     #
@@ -733,7 +851,7 @@ module Mabmapper
     # Notation
     #
     field :notation do
-      doc.field('700', ind2: ' ').subfield('a').get.values(join_subfields: true)
+      doc.field('700', ind2: ' ').subfield('a').get.values(join_subfields: '')
     end
 
     field :notation_sort do
@@ -761,7 +879,16 @@ module Mabmapper
       all_stack = fields.map{|f| f.subfields.find {|sf| sf.name == 'b' && sf.value.match(/02|03|04|07/)}.present?}.all?
 
       # Zeitschriftensignatur (haben Vorrgang, falls vorhanden)
-      signatures << doc.field('200', ind2: ' ').subfield('f').get.value.try(:gsub, ' ', '')
+      #
+      # Achtung, bei Feld 200 handelt es sich um einen Aleph-Expand. Dieses Feld ist an den beiden leeren Indikatoren zu erkennen.
+      # Darüber hinaus kann dieses Feld mehrfach vorkommen. Wir nehmen an, dass Subfeld 0 eine Art Zählung angibt, weshalb dort
+      # ein Wert von '1' zu bevorzugen ist.
+      #
+      signatures << doc.field('200', ind1: ' ', ind2: ' ').get.fields
+      .select { |f| f.get_subfield('f').present?                  }
+      .select { |f| (value = f.get_subfield('0').try(:value)) == '1' || value.nil? }
+      .map    { |f| f.get_subfield('f').value.try(:gsub, ' ', '') }
+      .first.presence
 
       # Wenn alle Exemplare im Magzin stehen, dann nimm nur die erste signatur
       if all_stack
@@ -803,6 +930,27 @@ module Mabmapper
       # Stücktitel Signatur
       signatures << doc.field('100', ind2: ' ').subfield('a').get.value
 
+      # Some additional love for journal signatures
+      signatures.map! do |signature|
+        # if this is a journal signature
+        if signature.try(:[], /\d+[A-Za-z]\d+$/).present?
+          # unless there is a leading standortkennziffer
+          unless signature.starts_with?('P')
+            standort_kennziffer = if (loc_standort_kennziffer = doc.field('LOC').subfield('b').get.value).present?
+              loc_standort_kennziffer
+            elsif (f105a = doc.field('105').subfield('a').get.value).present?
+              f105a
+            end
+
+            standort_kennziffer.present? ? "P#{standort_kennziffer}/#{signature}".gsub(/\/\//, '/') : signature
+          else
+            signature
+          end.downcase.capitalize # last but not least make journal signatures like P10/34T24 to P10/34t24
+        else
+          signature
+        end
+      end
+
       # Fertig. Wir nehmen die erste Signatur zur Anzeige
       signatures.flatten.map(&:presence).compact.uniq.first
     end
@@ -813,9 +961,10 @@ module Mabmapper
       # Stücktitel Signatur
       signatures << doc.field('100', ind2: ' ').subfield('a').get.value
       # Zeitschriftensignatur
-      signatures << doc.field('200', ind2: ' ').subfield('f').get.value
+      signatures << doc.field('200', ind1: ' ', ind2: ' ').subfield('f').get.values
 
-      signatures.flatten.map(&:presence).compact.map do |signature|
+      signatures = signatures.flatten.map(&:presence).compact
+      .map do |signature|
         _signature = signature
           .gsub(/\A\//, '') # remove leading '/' for some journal signatures
           .gsub(/\s+/,  '') # remove spaces for some journal signatures (e.g. 'P 10/34 t 26')
@@ -826,7 +975,31 @@ module Mabmapper
 
         # for journals which only have one single signature with leading 'Pxx/' like 'P10/34M3' create 'Pxx/'-less version also
         _signature_array.push _signature.gsub(/\AP\d+\//, '')
-      end.flatten.uniq
+      end.flatten
+
+      # if any signature is a journal signature
+      if (journal_signature = signatures.select { |signature| signature.try(:[], /\d+[A-Za-z]\d+$/).present? }.first).present?
+        if signatures.none? { |signature| signature.starts_with? 'P' }
+          # TODO: code duplication with :signature
+          standort_kennziffer = if (loc_standort_kennziffer = doc.field('LOC').subfield('b').get.value).present?
+            loc_standort_kennziffer
+          elsif (f105a = doc.field('105').subfield('a').get.value).present?
+            f105a
+          end
+
+          if standort_kennziffer.present?
+            signatures << "P#{standort_kennziffer}/#{journal_signature}".gsub(/\/\//, '/')
+          end
+        end
+      end
+
+      signatures.map! do |signature|
+        is_journal_signature = signature.match(/(P\d\d\/)?\d\d?[a-zA-Z]\d\d?/)
+        spaced_journal_signature = signature.gsub(/\AP(\d\d)/, 'P \1').gsub(/(\d\d?)([a-zA-Z])(\d\d?)/, '\1 \2 \3') if is_journal_signature
+        [signature, spaced_journal_signature]
+      end.flatten!
+
+      signatures.flatten.map(&:presence).compact.uniq
     end
 
     #
@@ -839,7 +1012,7 @@ module Mabmapper
     field :resource_link do
       fulltext_links = []
 
-      links = doc.field('655').subfield(['u', '3', 'z', 't']).get
+      links = doc.field('655').subfield(['u', '3', 'z', 't']).get.fields
       links.each do |link|
         url        = link.get_subfield('u').try(:value)
         subfield_3 = link.get_subfield('3') # HBZ Inhaltsverzeichnisse
@@ -862,7 +1035,7 @@ module Mabmapper
     field :link_to_toc do
       toc_links = []
 
-      links = doc.field('655').subfield(['u', '3', 'z', 't']).get
+      links = doc.field('655').subfield(['u', '3', 'z', 't']).get.fields
       links.each do |link|
         url        = link.get_subfield('u').try(:value)
         subfield_3 = link.get_subfield('3') # HBZ Inhaltsverzeichnisse
@@ -903,7 +1076,7 @@ module Mabmapper
       #   f - Signatur
       r = []
 
-      fields = doc.field('200', ind2: ' ').subfield(['0', 'a', 'b', 'c', 'e', 'f']).get
+      fields = doc.field('200', ind2: ' ').subfield(['0', 'a', 'b', 'c', 'e', 'f']).get.fields
       fields.each do |field|
         field_0 = field.get_subfield('0')
         field_a = field.get_subfield('a')
@@ -1032,19 +1205,37 @@ module Mabmapper
       ].select { |superorder| superorder[:label].present? }.map(&:to_json).presence
     end
 
+    field :local_comment do
+      doc.field('125', ind1: ' ', ind2: ' ').subfield(['_', 'a']).get.fields.map(&:values).flatten.uniq.presence
+    end
+
     #
-    # doc
+    # additional_data
+    # ( a complex data structure to be stored by the search engine; avoids to touch the normalization rules everytime )
     #
-
-    # a complex data structure to be stored by the search engine
-    field :doc do
-      doc = {
-        :redactional_remark => ref(:redactional_remark)
+    field :additional_data do
+      additional_data = {
+        author_statement:                   ref(:author_statement),
+        corporate_body_contributor_display: ref(:corporate_body_contributor_display),
+        corporate_body_creator_display:     ref(:corporate_body_creator_display),
+        local_comment:                      ref(:local_comment),
+        person_contributor_display:         ref(:person_contributor_display),
+        person_creator_display:             ref(:person_creator_display),
+        redactional_remark:                 ref(:redactional_remark)
+    
       }
       .inject({}) { |hash, (key, value)| hash[key] = value if value.present?; hash }
-
-      doc.to_json if doc.present?
+    
+      additional_data.to_json if additional_data.present?
     end
 
+    #
+    # mab
+    #
+    #field :mab do
+    #  (filtered_xml = doc.xml.clone).xpath('/OAI-PMH/ListRecords/record/metadata/record/datafield[@tag="TXT" or @tag="PLK" or @tag="PSW" or @tag="PPE"]').remove
+    #  Base64.strict_encode64(ActiveSupport::Gzip.compress(filtered_xml, Zlib::BEST_COMPRESSION)) unless filtered_xml.nil?
+    #end
+
   end
 end

data/lib/mabmapper/cli.rb

@@ -1,7 +1,6 @@
 #
 # The command line interface class
 #
-require 'rubygems/test_utilities'
 require 'mabmapper/elasticsearch_writer'
 require 'mabmapper/tar_writer'
 

data/lib/mabmapper/engine.rb

@@ -76,6 +76,8 @@ module Mabmapper
 
     # @see: http://www.dan-manges.com/blog/ruby-dsls-instance-eval-with-delegation
     class Field
+      attr_accessor :engine
+
       def initialize(name, &block)
         @name   = name.to_s
         @proc   = block

data/lib/mabmapper/mab_xml/document.rb

@@ -6,6 +6,8 @@ module Mabmapper
     class Document
       include QueryHelper
 
+      attr_accessor :xml
+
       def initialize(contents)
         @xml = Nokogiri::XML(contents)
         @xml.remove_namespaces!

data/lib/mabmapper/mab_xml/query_helper.rb

@@ -85,7 +85,7 @@ module Mabmapper
 
         options = [*value].map do |value|
           if value
-            negation = value.starts_with?('-') and value.length > 1
+            negation = value.starts_with?('-') && value.length > 1
             global_negation = true if negation
 
             negation ? "not(@#{name}='#{value.slice(1..-1)}')" : "@#{name}='#{value}'"

data/lib/mabmapper/version.rb

@@ -1,3 +1,3 @@
 module Mabmapper
-  VERSION = "1.0.0.pre15"
+  VERSION = "1.0.0.pre16"
 end

data/mabmapper.gemspec

@@ -27,7 +27,8 @@ Gem::Specification.new do |gem|
   gem.add_dependency('oj', '~> 2.1.4')
   gem.add_dependency('stringex', '~> 2.1.0')
 
-  gem.add_development_dependency('minitest', '~> 4.7.5')
-  gem.add_development_dependency('pry',      '~> 0.9.12.2')
-  gem.add_development_dependency('pry-nav',  '~> 0.2.3')
+  gem.add_development_dependency('minitest',         '~> 4.7.5')
+  gem.add_development_dependency('pry',              '0.9.12.2') # stuck to 0.9.12.2 due to repl color issue
+  gem.add_development_dependency('pry-nav',          '~> 0.2.3')
+  gem.add_development_dependency('pry-syntax-hacks', '~> 0.0.6')
 end