libis-tools 0.9.9 → 0.9.10

data/spec/metadata/marc21_parser_spec.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require 'libis/tools/metadata/parser/marc21_parser'
+require 'parslet/convenience'
+require 'pp'
+
+require_relative 'marc21_parser_data'
+
+describe 'MARC 21 parser' do
+  subject(:parser) { Libis::Tools::Metadata::Marc21Parser.new }
+
+  context 'Syntax parser' do
+
+    marc21_parser_testdata.each do |expectation|
+
+      next unless expectation.has_key?(:tree)
+
+      if expectation[:tree] != :failure
+        it "parses #{expectation[:title]}" do
+          expect {
+            # noinspection RubyArgCount,RubyUnusedLocalVariable
+            tree = parser.select.parse(expectation[:input]) if expectation[:title] =~ /^select /
+            # noinspection RubyArgCount,RubyUnusedLocalVariable
+            tree = parser.format.parse(expectation[:input]) if expectation[:title] =~ /^format /
+            # pp tree
+            expect(tree).to be_a Hash
+            expect(tree).to match expectation[:tree]
+          }.to_not raise_error
+        end
+      else
+        it "does not parse #{expectation[:title]}" do
+          expect {
+            # noinspection RubyArgCount,RubyUnusedLocalVariable
+            tree = parser.select.parse(expectation[:input]) if expectation[:title] =~ /^select /
+            # noinspection RubyArgCount,RubyUnusedLocalVariable
+            tree = parser.format.parse(expectation[:input]) if expectation[:title] =~ /^format /
+            # pp tree
+          }.to raise_error(Parslet::ParseFailed)
+        end
+      end
+
+    end
+  end
+
+  context 'Transformer' do
+
+    let(:transformer) { parser.transformer }
+
+    it 'can be created' do
+      expect(transformer).to be_a Libis::Tools::Metadata::Marc21Parser::Transformer
+    end
+
+    marc21_parser_testdata.each do |expectation|
+
+      next unless expectation.has_key?(:transform)
+
+      it "transforms #{expectation[:title]}" do
+        tree = parser.parse(expectation[:input])
+        # pp tree
+        expect(transformer.apply(tree)).to eq expectation[:transform]
+      end
+
+    end
+
+  end
+
+end

data/spec/metadata/marc21_spec.rb

@@ -0,0 +1,116 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require 'libis/tools/metadata/marc21_record'
+require 'libis/tools/xml_document'
+
+require 'rspec/matchers'
+require 'equivalent-xml'
+require 'libis/tools/extend/string'
+
+describe 'Marc21Record' do
+
+  subject(:record) { Libis::Tools::Metadata::Marc21Record.new(data) {} }
+  let(:data) { xml.root }
+
+  context '8389207' do
+    let(:xml) { Libis::Tools::XmlDocument.open(File.join(File.dirname(__FILE__), '8389207.marc')) }
+
+    it 'load from xml' do
+      expect(record.marc_dump).to eq <<-STR.align_left
+        LDR:'01068nam 2200241u 4500'
+        005:'20150701153710.0'
+        008:'000608m17221724xx |||| | 000|0 lat c'
+        001:'9921543960101471'
+        035: : :
+        \ta:["(BeLVLBS)002154396LBS01-Aleph"]
+        035: : :
+        \ta:["8389207"]
+        245:0:0:
+        \ta:["Anecdota Graeca, sacra et profana /"]
+        \tc:["ex codicibus manu exaratis nunc primum in lucem edita, versione Latina donata, et notis\\n      illustrata a Io. Christophoro Wolfio ... Tom. I [-IIII]\\n    "]
+        264: :1:
+        \ta:["Hamburgi"]
+        \tb:["apud Theodorum Christophorum Felginer,"]
+        \tc:["1722-1724"]
+        300: : :
+        \ta:["8o: 4 v.; [22], 298, [8]; [16], 354, [1]; [16], 299, [7]; [16], 271, [5] p."]
+        336: : :
+        \ta:["text"]
+        \t2:["rdacontent"]
+        337: : :
+        \ta:["unmediated"]
+        \t2:["rdamedia"]
+        338: : :
+        \ta:["volume"]
+        \t2:["rdacarrier"]
+        500: : :
+        \ta:["Ded. Petrus Theodorus Seelmann; Erdmannus Neumeister; Thomas Claussen; Joannes Grammius\\n    "]
+        500: : :
+        \ta:["Elk deel heeft eigen titelp. in roodzwartdruk, met drukkersmerk"]
+        650: :7:
+        \t2:["UDC"]
+        \ta:["276 =75"]
+        \tx:["Griekse patrologie"]
+        653: :6:
+        \ta:["Books before 1840"]
+        700:1: :
+        \ta:["Wolf, Johann Christoph"]
+        \td:["1683-1739"]
+        \t4:["aut"]
+        953: : :
+        \ta:["1701-1750"]
+        998: : :
+        \ta:["LBS01"]
+        \tb:["bib_200501.mrc.2.av"]
+        INST: : :
+        \ta:["32KUL_LIBIS_NETWORK"]
+        \tb:["P"]
+        \tc:["71134440820001471"]
+        INST: : :
+        \ta:["32KUL_KUL"]
+        \tb:["P"]
+        \tc:["21304345390001488"]
+        AVA: : :
+        \ta:["32KUL_KUL"]
+        \tb:["GBIB"]
+        \tc:["GBIB: Godgeleerdheid"]
+        \td:["276.030.4 WOLF Anec"]
+        \te:["available"]
+        \tf:["1"]
+        \tg:["0"]
+        \tj:["GBIB"]
+        \tp:["1"]
+        MMS: : :
+        \tb:["9921543960101471"]
+        \ta:["32KUL_LIBIS_NETWORK"]
+        MMS: : :
+        \tb:["9921543960101488"]
+        \ta:["32KUL_KUL"]
+      STR
+    end
+
+    it 'convert to dublin core' do
+      record.extend Libis::Tools::Metadata::Mappers::Kuleuven
+      xml_doc = Libis::Tools::XmlDocument.parse <<STR
+<?xml version="1.0" encoding="utf-8"?>
+<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:dcterms="http://purl.org/dc/terms/">
+  <dc:identifier>urn:ControlNumber:9921543960101471</dc:identifier>
+  <dc:identifier>(BeLVLBS)002154396LBS01-Aleph</dc:identifier>
+  <dc:identifier>8389207</dc:identifier>
+  <dc:title>Anecdota Graeca, sacra et profana /</dc:title>
+  <dc:creator>Wolf, Johann Christoph, 1683-1739, (author)</dc:creator>
+  <dc:description>Ded. Petrus Theodorus Seelmann; Erdmannus Neumeister; Thomas Claussen; Joannes Grammius\n    \nElk deel heeft eigen titelp. in roodzwartdruk, met drukkersmerk</dc:description>
+  <dc:date>1722 - 1724</dc:date>
+  <dcterms:extent>8o: 4 v.; [22], 298, [8]; [16], 354, [1]; [16], 299, [7]; [16], 271, [5] p.</dcterms:extent>
+  <dc:language/>
+</record>
+STR
+      expect(record.to_dc.root).to be_equivalent_to(xml_doc.root).respecting_element_order
+      record.to_dc.elements.each_with_index do |element, i|
+        expect(element).to be_equivalent_to(xml_doc.root.elements[i])
+      end
+    end
+  end
+
+end
+

data/spec/metadata/metadata_mapper_spec.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+require_relative '../spec_helper'
+require 'libis/tools/metadata/mapper'
+require 'libis/tools/metadata/parsers'
+require 'parslet'
+require 'parslet/convenience'
+require 'pp'
+
+$DEBUG = false
+
+describe 'Metadata Mapper' do
+
+  subject(:mapper) { Libis::Tools::Metadata::Mapper.new(
+      Libis::Tools::Metadata::Marc21Parser.new,
+      Libis::Tools::Metadata::DublinCoreParser.new,
+      Libis::Tools::Metadata::Marc21Parser.new,
+      File.join(File.dirname(__FILE__), '..', 'data', 'MetadataMapping.xlsx')) }
+
+  it 'Initialization' do
+    expect(mapper).to_not be_nil
+  end
+
+end

data/spec/spec_helper.rb

@@ -10,3 +10,16 @@ Bundler.setup
 require 'rspec'
 require 'libis-tools'
 
+# RSpec.configure do |config|
+#   original_stderr = $stderr
+#   original_stdout = $stdout
+#   config.before(:all) do
+#     # Redirect stderr and stdout
+#     $stderr = File.open(File::NULL, 'w')
+#     $stdout = File.open(File::NULL, 'w')
+#   end
+#   config.after(:all) do
+#     $stderr = original_stderr
+#     $stdout = original_stdout
+#   end
+# end

data/test.rb

@@ -0,0 +1,61 @@
+require 'libis-tools'
+require 'parslet'
+require 'parslet/convenience'
+
+# noinspection RubyResolve
+class FieldSpecParser < Libis::Tools::Metadata::BasicParser
+
+  root(:criteria)
+
+  rule(:criteria) { selection >> ( spaces >> selection ).repeat }
+
+  rule(:selection) { must >> must_not.maybe }
+
+  rule(:must) { names.as(:must).maybe >> (one_of | only_one_of).maybe }
+  rule(:must_not) { minus >> must.as(:not) }
+
+  rule(:one_of) { lrparen >> names.as(:one_of) >> rrparen }
+  rule(:only_one_of) { lcparen >> names.as(:only_one_of) >> rcparen }
+
+  rule(:names) { (character | number).repeat(1) }
+
+  def self.criteria_to_s(criteria)
+    case criteria
+      when Array
+        # leave as is
+      when Hash
+        criteria = [criteria]
+      else
+        return criteria
+    end
+    criteria.map { |selection|  selection_to_s(selection) }.join(' ')
+  end
+
+  def self.selection_to_s(selection)
+    return selection unless selection.is_a? Hash
+    result = "#{selection[:must]}"
+    result += "(#{selection[:one_of]})" if selection[:one_of]
+    result += "{#{selection[:only_one_of]}}" if selection[:only_one_of]
+    result += "-#{selection_to_s(selection[:not])}" if selection[:not]
+    result
+  end
+
+end
+
+require 'awesome_print'
+
+def parse(string)
+  tree = FieldSpecParser.new.parse_with_debug(string)
+  puts "parse '#{FieldSpecParser.criteria_to_s(tree)}'"
+  ap tree
+end
+
+parse ''
+parse 'abc'
+parse 'abc-de'
+parse 'abc(de)'
+parse 'abc{de}'
+parse 'a(bc)-de'
+parse 'a-b     c-d'
+parse 'a-(bc)'
+parse 'a-{bc}'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-tools
 version: !ruby/object:Gem::Version
-  version: 0.9.9
+  version: 0.9.10
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-22 00:00:00.000000000 Z
+date: 2015-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: backports
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +170,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: simple_xlsx_reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: '1.0'
 description: Some tool classes for other LIBIS gems.
 email:
 - kris.dekeyser@libis.be
@@ -186,19 +228,32 @@ files:
 - lib/libis/tools/command.rb
 - lib/libis/tools/config.rb
 - lib/libis/tools/config_file.rb
-- lib/libis/tools/dc_record.rb
 - lib/libis/tools/deep_struct.rb
 - lib/libis/tools/extend/empty.rb
 - lib/libis/tools/extend/hash.rb
+- lib/libis/tools/extend/kernel.rb
 - lib/libis/tools/extend/ostruct.rb
 - lib/libis/tools/extend/string.rb
 - lib/libis/tools/extend/struct.rb
 - lib/libis/tools/logger.rb
+- lib/libis/tools/metadata.rb
 - lib/libis/tools/metadata/dublin_core_record.rb
-- lib/libis/tools/metadata/field_spec.rb
+- lib/libis/tools/metadata/field_format.rb
 - lib/libis/tools/metadata/fix_field.rb
+- lib/libis/tools/metadata/mapper.rb
+- lib/libis/tools/metadata/mappers/flandrica.rb
+- lib/libis/tools/metadata/mappers/kuleuven.rb
 - lib/libis/tools/metadata/marc21_record.rb
 - lib/libis/tools/metadata/marc_record.rb
+- lib/libis/tools/metadata/parser/basic_parser.rb
+- lib/libis/tools/metadata/parser/dublin_core_parser.rb
+- lib/libis/tools/metadata/parser/marc21_parser.rb
+- lib/libis/tools/metadata/parser/marc_format_parser.rb
+- lib/libis/tools/metadata/parser/marc_rules.rb
+- lib/libis/tools/metadata/parser/marc_select_parser.rb
+- lib/libis/tools/metadata/parser/patch.rb
+- lib/libis/tools/metadata/parser/subfield_criteria_parser.rb
+- lib/libis/tools/metadata/parsers.rb
 - lib/libis/tools/metadata/sharepoint_mapping.rb
 - lib/libis/tools/metadata/sharepoint_record.rb
 - lib/libis/tools/metadata/var_field.rb
@@ -212,18 +267,26 @@ files:
 - spec/command_spec.rb
 - spec/config_file_spec.rb
 - spec/config_spec.rb
+- spec/data/MetadataMapping.xlsx
 - spec/data/test.data
 - spec/data/test.xml
 - spec/data/test.yml
 - spec/data/test_config.yml
 - spec/deep_struct_spec.rb
 - spec/logger_spec.rb
+- spec/metadata/8389207.marc
+- spec/metadata/dublin_core_parser_spec.rb
 - spec/metadata/dublin_core_spec.rb
+- spec/metadata/marc21_parser_data.rb
+- spec/metadata/marc21_parser_spec.rb
+- spec/metadata/marc21_spec.rb
+- spec/metadata/metadata_mapper_spec.rb
 - spec/parameter_container_spec.rb
 - spec/parameter_spec.rb
 - spec/spec_helper.rb
 - spec/test.xsd
 - spec/xmldocument_spec.rb
+- test.rb
 - test/test_helper.rb
 - test/webservices/test_ca_item_info.rb
 - test/webservices/test_ca_search.rb
@@ -247,7 +310,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: LIBIS toolbox.
@@ -257,13 +320,20 @@ test_files:
 - spec/command_spec.rb
 - spec/config_file_spec.rb
 - spec/config_spec.rb
+- spec/data/MetadataMapping.xlsx
 - spec/data/test.data
 - spec/data/test.xml
 - spec/data/test.yml
 - spec/data/test_config.yml
 - spec/deep_struct_spec.rb
 - spec/logger_spec.rb
+- spec/metadata/8389207.marc
+- spec/metadata/dublin_core_parser_spec.rb
 - spec/metadata/dublin_core_spec.rb
+- spec/metadata/marc21_parser_data.rb
+- spec/metadata/marc21_parser_spec.rb
+- spec/metadata/marc21_spec.rb
+- spec/metadata/metadata_mapper_spec.rb
 - spec/parameter_container_spec.rb
 - spec/parameter_spec.rb
 - spec/spec_helper.rb