libis-tools 0.9.9 → 0.9.10

data/lib/libis/tools/metadata/marc21_record.rb

@@ -14,7 +14,7 @@ module Libis
 
         def get_all_records
 
-          @all_records = Hash.new { |h, k| h[k] = [] }
+          @all_records.clear
 
           @node.xpath('.//leader').each { |f|
             @all_records['LDR'] << FixField.new('LDR', f.content)
@@ -31,10 +31,11 @@ module Libis
             tag = v['tag']
             tag = '%03d' % tag.to_i if tag.size < 3
 
-            subfields = Hash.new { |h, k| h[k] = [] }
-            v.xpath('.//subfield').each { |s| subfields[s['code']] << CGI::escapeHTML(s.content) }
+            varfield = VarField.new(tag, v['ind1'].to_s, v['ind2'].to_s)
 
-            @all_records[tag] << VarField.new(tag, v['ind1'].to_s, v['ind2'].to_s, subfields)
+            v.xpath('.//subfield').each { |s| varfield.add_subfield(s['code'], CGI::escapeHTML(s.content)) }
+
+            @all_records[tag] << varfield
 
           }
 

data/lib/libis/tools/metadata/marc_record.rb

@@ -8,58 +8,105 @@ require 'libis/tools/assert'
 
 require_relative 'fix_field'
 require_relative 'var_field'
-require_relative 'field_spec'
+require_relative 'field_format'
 
 module Libis
   module Tools
     module Metadata
 
       # noinspection RubyTooManyMethodsInspection
+
+      # Base class for reading MARC based records.
+      #
+      # For indicator selection: '#' or '' (empty) is wildcard; '_' or ' ' (space) is blank.
       class MarcRecord
 
+        # Create a new MarcRecord object
+        #
+        # @param [XML node] xml_node XML node from Nokogiri or XmlDocument that contains child nodes with the data for
+        #         one MARC record.
         def initialize(xml_node)
           @node = xml_node
+          @node.document.remove_namespaces!
+          @all_records = Hash.new { |h, k| h[k] = Array.new }
         end
 
+        # Access to the XML node that was supplied to the constructor
+        # @return [XML node]
         def to_raw
           @node
         end
 
+        # Returns the internal data structure (a Hash) with all the MARC data.
+        #
+        # The internal structure is a Hash with the tag as key and as value an Array of either FixField or VarField
+        # instances.
+        #
+        # @return [Hash] internal data structure
         def all
-          # noinspection RubyResolve
-          @all_records ||= get_all_records
+          return @all_records unless @all_records.empty?
+          @all_records = get_all_records
         end
 
+        # Iterates over all the MARC fields.
+        #
+        # If a block is supplied it will be called for each field in the MARC record. The supplied argument will be the
+        # FixField or VarField instance for each field.
+        #
+        # @return [Array] The list of the field data or return values for each block call.
         def each
-          all.each do |k, v|
-            yield k, v
+          all.map { |_, field_array| field_array }.flatten.map do |field|
+            block_given? ? yield(field) : field
           end
         end
 
-        def all_tags(tag, subfields = '')
-          tag_, ind1, ind2 = tag =~ /^\d{3}/ ? [tag[0..2], tag[3], tag[4]] : [tag, nil, nil]
-          result = get_records(tag_, ind1, ind2, subfields)
+        # Get all fields matching search criteria.
+        #
+        # A block with one parameter can be supplied when calling this method. Each time a match is found, the block
+        # will be called with the field data as argument and the return value of the block will be added to the method's
+        # return value. This could for example be used to narrow the selection of the fields:
+        #
+        #     # Only select 700 tags where $4 subfield contains 'abc', 'def' or 'xyz'
+        #     record.all_tags('700') { |v| v.subfield['4'] =~ /^(abc|def|xyz)$/ ? v : nil }.compact
+        #
+        # @param [String] tag Tag selection string. Tag name with indicators, '#' for wildcard, '_' for blank. If an
+        #     extra subfield name is added, a result will be created for each instance found of that subfield.
+        # @param [String] subfields Subfield specification. See FieldFormat class for more info; ignored for controlfields.
+        # @param [Proc] select_block block that will be executed once for each field found. The block takes one argument
+        #     (the field) and should return true or false. True selects the field, false rejects it.
+        # @return [Array] If a block was supplied to the method call, the array will contain the result of the block
+        #     for each tag found. Otherwise the array will just contain the data for each matching tag.
+        def all_tags(tag, subfields = '', select_block = Proc.new { |_| true})
+          t, ind1, ind2, subfield = tag =~ /^\d{3}/ ? [tag[0..2], tag[3], tag[4], tag[5]] : [tag, nil, nil, nil]
+          result = get_records(t, ind1, ind2, subfield, subfields, &select_block)
           return result unless block_given?
           result.map { |record| yield record }
-          result.size > 0
         end
 
-        def first_tag(t, subfields = '')
-          result = all_tags(t, subfields).first
-          return result unless block_given?
-          return false unless result
-          yield result
-          true
+        alias_method :each_tag, :all_tags
+
+        def select_fields(tag, select_block = nil, &block)
+          all_tags(tag, nil, select_block, &block)
         end
 
-        def each_tag(t, s = '')
-          all_tags(t, s).each do |record|
-            yield record
-          end
+        # Find the first field matching the criteria.
+        #
+        # If a block is supplied, it will be called with the found field data. The return value will be whatever the
+        # block returns. If no block is supplied, the field data will be returned. If nothing was found, the return
+        # value is nil.
+        #
+        # @param [String] tag Tag selection string. Tag name with indicators, '#' for wildcard, '_' for blank.
+        # @param [String] subfields Subfield specification. See FieldFormat class for more info; ignored for controlfields.
+        # @return [Object] nil if nothing found; field data or whatever block returns.
+        def first_tag(tag, subfields = '')
+          result = all_tags(tag, subfields).first
+          return nil unless result
+          return result unless block_given?
+          yield result
         end
 
-        def all_fields(t, s)
-          r = all_tags(t, s).collect { |tag| tag.fields_array(s) }.flatten.compact
+        def all_fields(tag, subfields)
+          r = all_tags(tag, subfields).collect { |tag| tag.subfields_array(subfields) }.flatten.compact
           return r unless block_given?
           r.map { |field| yield field }
           r.size > 0
@@ -116,11 +163,11 @@ module Libis
           record = ''
           doc_number = tag('001').datas
 
-          all.select { |t| t.is_a? FixField }.each { |t| record += "#{format('%09s', doc_number)} #{t.tag}   L #{t.datas}\n" }
-          all.select { |t| t.is_a? VarField }.each { |t|
+          all.select { |t| t.is_a? Libis::Tools::Metadata::FixField }.each { |t| record += "#{format('%09s', doc_number)} #{t.tag}   L #{t.datas}\n" }
+          all.select { |t| t.is_a? Libis::Tools::Metadata::VarField }.each { |t|
             record += "#{format('%09s', doc_number)} #{t.tag}#{t.ind1}#{t.ind2} L "
             t.keys.each { |k|
-              t.field_array(k).each { |f|
+              t.subfield_array(k).each { |f|
                 record += "$$#{k}#{CGI::unescapeHTML(f)}"
               }
             }
@@ -134,37 +181,37 @@ module Libis
 
         def element(*parts)
           opts = options parts
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def list_s(*parts)
           opts = options parts, join: ' '
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def list_c(*parts)
           opts = options parts, join: ', '
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def list_d(*parts)
           opts = options parts, join: ' - '
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def repeat(*parts)
           opts = options parts, join: '; '
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def opt_r(*parts)
           opts = options parts, fix: '()'
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def opt_s(*parts)
           opts = options parts, fix: '[]'
-          field_spec(opts, *parts)
+          field_format(opts, *parts)
         end
 
         def odis_link(group, id, label)
@@ -177,11 +224,11 @@ module Libis
           default.merge(args.last.is_a?(::Hash) ? args.pop : {})
         end
 
-        def field_spec(default_options, *parts)
-          FieldSpec.new(*parts).add_default_options(default_options).to_s
+        def field_format(default_options, *parts)
+          Libis::Tools::Metadata::FieldFormat.new(*parts).add_default_options(default_options).to_s
         end
 
-        def get_records(tag, ind1 = '', ind2 = '', subfields = '')
+        def get_records(tag, ind1 = '', ind2 = '', subfield = nil, subfields = '', &block)
 
           ind1 ||= ''
           ind2 ||= ''
@@ -193,14 +240,26 @@ module Libis
           ind2.tr!('_', ' ')
           ind2.tr!('#', '')
 
-          all[tag].select do |v|
-            v.is_a?(FixField) ||
+          found = all[tag].select do |v|
+            result = v.is_a?(Libis::Tools::Metadata::FixField) ||
                 ((ind1.empty? or v.ind1 == ind1) &&
                     (ind2.empty? or v.ind2 == ind2) &&
-                    v.match_fieldspec?(subfields)
+                    v.match(subfields)
                 )
+            result &&= block.call(v) if block
+            result
           end
 
+          return found unless subfield
+
+          # duplicate tags for subfield instances
+          found.map do |field|
+            next unless field.is_a? Libis::Tools::Metadata::FixField
+            field.subfield_data[subfield].map do |sfield|
+              field.dup.subfield_data[subfield] = [sfield]
+            end
+          end.compact.flatten
+
         end
 
       end

data/lib/libis/tools/metadata/parser/basic_parser.rb

@@ -0,0 +1,118 @@
+# encoding: utf-8
+
+require 'parslet'
+require 'parslet/convenience'
+
+module Libis
+  module Tools
+    module Metadata
+      # noinspection RubyResolve
+      class BasicParser < Parslet::Parser
+        # space
+        rule(:space) { match('\s') }
+        rule(:space?) { space.maybe }
+        rule(:spaces) { space.repeat(1) }
+        rule(:spaces?) { space.repeat }
+
+        # numbers
+        rule(:number) { match('[0-9]') }
+        rule(:number?) { number.maybe }
+        rule(:integer) { number.repeat(1) }
+
+        # chars
+        rule(:character) { match(/[a-z]/i) }
+        rule(:character?) { character.maybe }
+        rule(:characters) { character.repeat(1) }
+
+        # word
+        rule(:wordchar) { match('\w') }
+
+        # name
+        rule(:name_string) { ((character | underscore) >> wordchar.repeat).repeat(1) }
+
+        # text
+        rule(:other) { not_paren }
+        rule(:text) { other.repeat(1) }
+        rule(:text?) { text.maybe }
+
+        # special chars
+        rule(:minus) { str('-') }
+        rule(:colon) { str(':') }
+        rule(:semicolon) { str(';') }
+        rule(:underscore) { str('_') }
+        rule(:hashtag) { str('#') }
+        rule(:dollar) { str('$') }
+        rule(:star) { str('*') }
+
+        # grouping
+        rule(:paren) { lparen | rparen }
+        rule(:lparen) { lrparen | lsparen | lcparen | squote | dquote }
+        rule(:rparen) { rrparen | rsparen | rcparen | squote | dquote }
+
+        rule(:not_paren) { paren.absent? >> any }
+        rule(:not_lparen) { lrparen.absent? >> lsparen.absent? >> lcparen.absent? >> squote.absent? >> dquote.absent? >> any }
+        rule(:not_rparen) { rrparen.absent? >> rsparen.absent? >> rcparen.absent? >> squote.absent? >> dquote.absent? >> any }
+
+        rule(:lrparen) { str('(') }
+        rule(:lsparen) { str('[') }
+        rule(:lcparen) { str('{') }
+        rule(:rrparen) { str(')') }
+        rule(:rsparen) { str(']') }
+        rule(:rcparen) { str('}') }
+
+        rule(:squote) { str("'") }
+        rule(:dquote) { str('"') }
+        rule(:quote) { squote | dquote }
+
+        rule(:not_squote) { squote.absent? >> any }
+        rule(:not_dquote) { dquote.absent? >> any }
+        rule(:not_quote) { quote.absent? >> any }
+
+        def complement(char)
+          case char
+            when '('
+              ')'
+            when '{'
+              '}'
+            when '['
+              ']'
+            else
+              char
+          end
+        end
+
+        def grouped(foo, left_paren = lparen)
+          scope {
+            left_paren.capture(:paren).as(:lparen) >>
+                foo >>
+                dynamic { |_, c| str(complement(c.captures[:paren])) }.as(:rparen)
+          }
+        end
+
+        def grouped_anonymous(foo, left_paren = lparen)
+          scope {
+            left_paren.capture(:paren) >>
+                foo >>
+                dynamic { |_, c| str(complement(c.captures[:paren])) }
+          }
+        end
+
+        def any_quoted(key = :text)
+          scope {
+            quote.capture(:quote) >>
+                dynamic { |_, c| (str(c.captures[:quote]).absent? >> any).repeat(1) }.maybe.as(key) >>
+                dynamic { |_, c| str(c.captures[:quote]) }
+          }
+        end
+
+        def transformer
+          self.class::Transformer.new rescue nil
+        end
+
+      end
+
+    end
+  end
+end
+
+

data/lib/libis/tools/metadata/parser/dublin_core_parser.rb

@@ -0,0 +1,36 @@
+# encoding: utf-8
+
+require 'parslet'
+
+require_relative 'basic_parser'
+
+module Libis
+  module Tools
+    module Metadata
+
+      # noinspection RubyResolve
+      class DublinCoreParser < Libis::Tools::Metadata::BasicParser
+        rule(:namespace) { match('[^:]').repeat(1).as(:namespace) >> str(':') }
+        rule(:namespace?) { namespace.maybe }
+
+        rule(:attribute) { namespace? >> name_string.as(:name) >> str('=') >> str('"') >> match('[^"]').repeat(1).as(:value) >> str('"') }
+        rule(:attributes) { attribute >> (spaces >> attribute).repeat }
+        rule(:attributes?) { attributes.maybe }
+        rule(:element) { name_string.as(:element) }
+        rule(:DC) { namespace >> element >> (spaces >> attributes.as(:attributes)).maybe }
+
+        root(:DC)
+
+        def to_target(tree)
+          tree = tree[:DC]
+          return nil unless tree
+          result = "xml['#{tree[:namespace]}'].#{tree[:element]}("
+          tree[:attributes].each { |attribute| result += "'#{attribute[:name]}' => '#{attribute[:value]}'" }
+          result + ').text'
+        end
+
+      end
+
+    end
+  end
+end

data/lib/libis/tools/metadata/parser/marc21_parser.rb

@@ -0,0 +1,206 @@
+# encoding: utf-8
+
+require 'parslet'
+
+require_relative 'basic_parser'
+require_relative 'marc_rules'
+
+module Libis
+  module Tools
+    module Metadata
+
+      # noinspection RubyResolve
+      class Marc21Parser < Libis::Tools::Metadata::BasicParser
+
+        root(:marc21)
+        rule(:marc21) { select.as(:select) | format.as(:format) }
+
+        # select syntax
+        rule(:select) do
+          str('MARC') >>
+              spaces? >> tag.as(:tag) >>
+              spaces? >> indicator.maybe.as(:ind1) >> indicator.maybe.as(:ind2) >>
+              spaces? >> subfield.maybe.as(:subfield) >>
+              spaces? >> condition.maybe.as(:condition)
+        end
+        rule(:condition) { grouped_anonymous(cond_format.as(:cond_format)) }
+        rule(:cond_format) { cond_entry.repeat(1).maybe.as(:entry) >> postfix.maybe.as(:postfix) }
+        rule(:cond_entry) { sf_reference | method_call | cond_group }
+        rule(:cond_group) { (prefix.maybe.as(:prefix) >> grouped(cond_format)).as(:cond_group) }
+
+        # Formatting syntax
+        rule(:format) { entry.repeat(1).maybe.as(:entry) >> postfix.maybe.as(:postfix) }
+
+        rule(:entry) { sf_reference | method_call | group }
+        # noinspection RubyArgCount
+        rule(:group) { (prefix.maybe.as(:prefix) >> grouped(format)).as(:group) }
+        # noinspection RubyArgCount
+        rule(:method_call) { (prefix.maybe.as(:prefix) >> sf_indicator >> grouped_anonymous(format, lrparen)).as(:method_call) }
+
+        # pre- and postfix
+        rule(:prefix) { other.repeat(1) }
+        rule(:prefix) { text }
+        rule(:postfix) { other.repeat(1) }
+        rule(:postfix) { text }
+
+        # subfield reference
+        rule(:sf_reference) { sf_variable.as(:subfield) | sf_fixed.as(:fixfield) }
+
+        rule(:sf_variable) { prefix.maybe.as(:prefix) >> sf_indicator >> sf_repeat.maybe.as(:repeat) >> sf_name }
+        rule(:sf_repeat) { star >> any_quoted(:separator).maybe }
+
+        rule(:sf_fixed) { prefix.maybe.as(:prefix) >> sf_indicator >> lsparen >> (sf_range | sf_position | sf_star) >> rsparen }
+        rule(:sf_star) { star.as(:all) }
+        rule(:sf_position) { integer.as(:position) }
+        rule(:sf_range) { integer.as(:first) >> minus >> integer.as(:last) }
+
+        rule(:other) { paren.absent? >> dollar.absent? >> any | litteral_dollar }
+
+        # tag
+        rule(:tag) { tag_numeric | tag_alpha }
+        rule(:tag_numeric) { number.repeat(3, 3) }
+        rule(:tag_alpha) { character.repeat(3, 3) }
+
+        # indicator
+        rule(:indicator) { hashtag | underscore | number | character }
+
+        # subfield
+        rule(:sf_indicator) { dollar }
+        rule(:sf_name) { (character | number).as(:name) }
+        rule(:sf_names) { (character | number).repeat(1).as(:names) }
+        rule(:subfield) { sf_indicator >> sf_name }
+        rule(:litteral_dollar) { dollar >> dollar }
+
+        # noinspection RubyResolve
+        class Transformer < Parslet::Transform
+          rule(name: simple(:name)) { "#{name}" }
+          # select transformation rules
+          rule(cond_group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: simple(:entry),
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "#{prefix}#{lparen}#{entry}#{postfix}#{rparen}"
+          }
+          rule(cond_group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: sequence(:entry),
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "#{prefix}#{lparen}#{entry.join}#{postfix}#{rparen}"
+          }
+          rule(cond_format: {
+                   entry: sequence(:entry),
+                   postfix: simple(:postfix)
+               }) { ", Proc.new { |f| #{entry.join}#{postfix} }" }
+          rule(select: {
+                   tag: simple(:tag),
+                   ind1: simple(:ind1),
+                   ind2: simple(:ind2),
+                   subfield: simple(:subfield),
+                   condition: simple(:condition)
+               }) { "record.select_fields('#{tag}#{ind1 || '#'}#{ind2 || '#'}#{subfield}'#{condition || ''})" }
+          # format transformation rules
+          rule(format: {
+                   entry: sequence(:entries),
+                   postfix: simple(:postfix)
+               }) do
+            if entries.size == 1 && postfix.nil?
+              entries.first
+            else
+              "field_format(#{entries.join(',')}#{", postfix: '#{postfix}'" if postfix}).to_s"
+            end
+          end
+          rule(group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: nil,
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "#{prefix}#{lparen}#{postfix}#{rparen}"
+          }
+          rule(group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: '',
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "#{prefix}#{lparen}#{postfix}#{rparen}"
+          }
+          rule(group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: simple(:entry),
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "field_format(#{entry}#{", prefix: '#{prefix}#{lparen}'" if prefix || lparen}#{", postfix: '#{postfix}#{rparen}'" if postfix || rparen}).to_s"
+          }
+          rule(group: {
+                   prefix: simple(:prefix),
+                   lparen: simple(:lparen),
+                   entry: sequence(:entries),
+                   postfix: simple(:postfix),
+                   rparen: simple(:rparen)}) {
+            "field_format(#{entries.join(',')}#{", prefix: '#{prefix}#{lparen}'" if prefix || lparen}#{", postfix: '#{postfix}#{rparen}'" if postfix || rparen}).to_s"
+          }
+          rule(fixfield: {
+                   prefix: nil,
+                   all: '*'
+               }) { 'f[]' }
+          rule(fixfield: {
+                   prefix: simple(:prefix),
+                   all: '*'
+               }) { "field_format(f[], prefix: '#{prefix}').to_s" }
+          rule(fixfield: {
+                   prefix: nil,
+                   position: simple(:position)
+               }) { "f[#{position}]" }
+          rule(fixfield: {
+                   prefix: simple(:prefix),
+                   position: simple(:position)
+               }) do
+            if prefix
+              "field_format(f[#{position}], prefix: '#{prefix}').to_s"
+            else
+              "f[#{position}]"
+            end
+          end
+          rule(fixfield: {
+                   prefix: nil,
+                   first: simple(:from),
+                   last: simple(:to)
+               }) { "f[#{from},#{to}]" }
+          rule(fixfield: {
+                   prefix: simple(:prefix),
+                   first: simple(:from),
+                   last: simple(:to)
+               }) { "field_format(f[#{from},#{to}], prefix: '#{prefix}').to_s" }
+          rule(subfield: {
+                   prefix: simple(:prefix),
+                   repeat: nil,
+                   name: simple(:name),
+               }) { "field_format(f.subfield('#{name}'), prefix: '#{prefix}').to_s" }
+          rule(subfield: {
+                   prefix: simple(:prefix),
+                   repeat: {separator: simple(:separator)},
+                   name: simple(:name),
+               }) { "field_format(f.subfield_array('#{name}')#{", prefix: '#{prefix}'" if prefix}, join: '#{separator}').to_s" }
+          rule(subfield: {
+                   prefix: simple(:prefix),
+                   repeat: '*',
+                   name: simple(:name),
+               }) { "field_format(f.subfield_array('#{name}')#{", prefix: '#{prefix}'" if prefix}, join: ';').to_s" }
+          rule(subfield: {
+                   prefix: nil,
+                   repeat: nil,
+                   name: simple(:name),
+               }) { "f.subfield('#{name}')" }
+        end
+
+      end
+
+    end
+  end
+end