libis-tools 0.9.20 → 0.9.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +36 -233
  3. data/Rakefile +5 -0
  4. data/lib/libis/tools.rb +1 -0
  5. data/lib/libis/tools/assert.rb +11 -0
  6. data/lib/libis/tools/checksum.rb +22 -5
  7. data/lib/libis/tools/command.rb +24 -3
  8. data/lib/libis/tools/config.rb +61 -33
  9. data/lib/libis/tools/config_file.rb +0 -1
  10. data/lib/libis/tools/deep_struct.rb +10 -2
  11. data/lib/libis/tools/extend/empty.rb +2 -2
  12. data/lib/libis/tools/extend/hash.rb +37 -18
  13. data/lib/libis/tools/extend/kernel.rb +9 -0
  14. data/lib/libis/tools/extend/string.rb +17 -8
  15. data/lib/libis/tools/logger.rb +95 -44
  16. data/lib/libis/tools/metadata.rb +5 -1
  17. data/lib/libis/tools/metadata/dublin_core_record.rb +22 -4
  18. data/lib/libis/tools/metadata/field_format.rb +49 -9
  19. data/lib/libis/tools/metadata/fix_field.rb +5 -0
  20. data/lib/libis/tools/metadata/mapper.rb +2 -1
  21. data/lib/libis/tools/metadata/mappers/flandrica.rb +8 -1
  22. data/lib/libis/tools/metadata/mappers/kuleuven.rb +6 -2
  23. data/lib/libis/tools/metadata/marc21_record.rb +1 -0
  24. data/lib/libis/tools/metadata/marc_record.rb +31 -12
  25. data/lib/libis/tools/metadata/parser/basic_parser.rb +2 -0
  26. data/lib/libis/tools/metadata/parser/dublin_core_parser.rb +2 -1
  27. data/lib/libis/tools/metadata/parser/marc21_parser.rb +2 -1
  28. data/lib/libis/tools/metadata/parser/marc_format_parser.rb +2 -1
  29. data/lib/libis/tools/metadata/parser/marc_rules.rb +2 -1
  30. data/lib/libis/tools/metadata/parser/marc_select_parser.rb +2 -1
  31. data/lib/libis/tools/metadata/parser/patch.rb +1 -0
  32. data/lib/libis/tools/metadata/parser/subfield_criteria_parser.rb +2 -1
  33. data/lib/libis/tools/metadata/sharepoint_mapping.rb +1 -0
  34. data/lib/libis/tools/metadata/sharepoint_record.rb +2 -0
  35. data/lib/libis/tools/metadata/var_field.rb +8 -0
  36. data/lib/libis/tools/mets_dnx.rb +61 -0
  37. data/lib/libis/tools/mets_file.rb +87 -604
  38. data/lib/libis/tools/mets_objects.rb +534 -0
  39. data/lib/libis/tools/parameter.rb +144 -21
  40. data/lib/libis/tools/thread_safe.rb +31 -0
  41. data/lib/libis/tools/version.rb +1 -1
  42. data/lib/libis/tools/xml_document.rb +18 -24
  43. data/libis-tools.gemspec +6 -2
  44. data/spec/config_spec.rb +3 -4
  45. data/spec/logger_spec.rb +13 -30
  46. data/spec/mets_file_spec.rb +17 -17
  47. metadata +53 -7
@@ -2,7 +2,6 @@
2
2
  require 'singleton'
3
3
  require 'yaml'
4
4
  require 'erb'
5
- require 'logger'
6
5
 
7
6
  require 'libis/tools/deep_struct'
8
7
 
@@ -4,10 +4,18 @@ require 'recursive-open-struct'
4
4
  module Libis
5
5
  module Tools
6
6
 
7
- # A class that derives from OpenStruct through the RecursiveOpenStruct. A RecursiveOpenStruct is derived from
8
- # stdlib's OpenStruct, but can be made recursive. DeepStruct enforces this behaviour and adds a clear! method.
7
+ # A class that derives from OpenStruct through the RecursiveOpenStruct.
8
+ # By wrapping a Hash recursively, it allows for easy access to the content by method names.
9
+ # A RecursiveOpenStruct is derived from stdlib's OpenStruct, but can be made recursive.
10
+ # DeepStruct enforces this behaviour and adds a clear! method.
9
11
  class DeepStruct < RecursiveOpenStruct
10
12
 
13
+ # Create a new DeepStruct from a Hash and configure the behaviour.
14
+ #
15
+ # @param [Hash] hash the initial data structure.
16
+ # @param [Hash] opts optional configuration options:
17
+ # * recurse_over_arrays: also wrap the Hashes that are enbedded in Arrays. Default: true.
18
+ # * preserver_original_keys: creating a Hash from the wrapper preserves symbols and strings as keys. Default: true.
11
19
  def initialize(hash = {}, opts = {})
12
20
  hash = {} unless hash
13
21
  opts = {} unless opts
@@ -1,6 +1,6 @@
1
- # encoding: utf-8
2
-
1
+ # Extension for NilClass
3
2
  class NilClass
3
+ # Allows nil.empty?
4
4
  def empty?
5
5
  true
6
6
  end
@@ -1,11 +1,14 @@
1
1
  require 'backports/rails/hash'
2
2
 
3
+ # Extension class for Hash
3
4
  class Hash
4
5
 
6
+ # Removes all hash entries for which value.empty? is true
5
7
  def cleanup
6
8
  self.delete_if { |_,v| v.nil? || (v.respond_to?(:empty?) ? v.empty? : false) }
7
9
  end unless method_defined? :cleanup
8
10
 
11
+ # Removes all hash entries for which value.empty? is true. Performed recursively.
9
12
  def recursive_cleanup
10
13
  delete_proc = Proc.new do |_, v|
11
14
  v.delete_if(&delete_proc) if v.kind_of?(Hash)
@@ -14,6 +17,7 @@ class Hash
14
17
  self.delete_if &delete_proc
15
18
  end unless method_defined? :recursive_cleanup
16
19
 
20
+ # Merges two hashes, but does so recursively.
17
21
  def recursive_merge(other_hash)
18
22
  self.merge(other_hash) do |_, old_val, new_val|
19
23
  if old_val.is_a? Hash
@@ -24,6 +28,7 @@ class Hash
24
28
  end
25
29
  end unless method_defined? :recursive_merge
26
30
 
31
+ # Merges two hashes in-place, but does so recursively.
27
32
  def recursive_merge!(other_hash)
28
33
  self.merge!(other_hash) do |_, old_val, new_val|
29
34
  if old_val.is_a? Hash
@@ -34,30 +39,38 @@ class Hash
34
39
  end
35
40
  end unless method_defined? :recursive_merge!
36
41
 
37
- def key_strings_to_symbols!(opts = {})
38
- self.replace self.key_strings_to_symbols opts
42
+ # Convert all keys to symbols. In-place operation.
43
+ # @param (see #key_strings_to_symbols)
44
+ def key_strings_to_symbols!(options = {})
45
+ self.replace self.key_strings_to_symbols options
39
46
  end unless method_defined? :key_strings_to_symbols!
40
47
 
41
- def key_strings_to_symbols(opts = {})
42
- opts = {resursive: false, upcase: false, downcase: false}.merge opts
48
+ # Return new Hash with all keys converted to symbols.
49
+ # @param [Hash] opts valid options are:
50
+ # * recursive : perform operation recursively
51
+ # * upcase : convert all keys to upper case
52
+ # * downcase : convert all keys to lower case
53
+ # all options are false by default
54
+ def key_strings_to_symbols(options = {})
55
+ options = {resursive: false, upcase: false, downcase: false}.merge options
43
56
 
44
57
  r = Hash.new
45
58
  self.each_pair do |k,v|
46
59
 
47
60
  k = k.to_s if k.kind_of? Symbol
48
61
  if k.kind_of? String
49
- k = k.downcase if opts[:downcase]
50
- k = k.upcase if opts[:upcase]
62
+ k = k.downcase if options[:downcase]
63
+ k = k.upcase if options[:upcase]
51
64
  k = k.to_sym
52
65
  end
53
66
 
54
- if opts[:recursive]
67
+ if options[:recursive]
55
68
  case v
56
69
  when Hash
57
- v = v.key_strings_to_symbols opts
70
+ v = v.key_strings_to_symbols options
58
71
  when Array
59
72
  # noinspection RubyResolve
60
- v = v.collect { |a| (a.kind_of? Hash) ? a.key_strings_to_symbols(opts) : Marshal.load(Marshal.dump(a)) }
73
+ v = v.collect { |a| (a.kind_of? Hash) ? a.key_strings_to_symbols(options) : Marshal.load(Marshal.dump(a)) }
61
74
  else
62
75
  # noinspection RubyResolve
63
76
  v = Marshal.load(Marshal.dump(v))
@@ -71,12 +84,18 @@ class Hash
71
84
  r
72
85
  end unless method_defined? :key_strings_to_symbols
73
86
 
74
- def key_symbols_to_strings!(opts = {})
75
- self.replace self.key_symbols_to_strings opts
87
+ # Convert all keys to strings. In-place operation.
88
+ # (@see #key_symbols_to_strings)
89
+ # @param (see #key_symbols_to_strings)
90
+ def key_symbols_to_strings!(options = {})
91
+ self.replace self.key_symbols_to_strings options
76
92
  end unless method_defined? :key_symbols_to_strings!
77
93
 
78
- def key_symbols_to_strings(opts = {})
79
- opts = {resursive: false, upcase: false, downcase: false}.merge opts
94
+ # Return new Hash with all keys converted to strings.
95
+ # (see #key_strings_to_symbols)
96
+ # @param (see #key_strings_to_symbols)
97
+ def key_symbols_to_strings(options = {})
98
+ options = {resursive: false, upcase: false, downcase: false}.merge options
80
99
 
81
100
  r = Hash.new
82
101
  self.each_pair do |k,v|
@@ -84,17 +103,17 @@ class Hash
84
103
  k = k.to_sym if k.kind_of? String
85
104
  if k.kind_of? Symbol
86
105
  k = k.to_s
87
- k = k.downcase if opts[:downcase]
88
- k = k.upcase if opts[:upcase]
106
+ k = k.downcase if options[:downcase]
107
+ k = k.upcase if options[:upcase]
89
108
  end
90
109
 
91
- if opts[:recursive]
110
+ if options[:recursive]
92
111
  case v
93
112
  when Hash
94
- v = v.key_symbols_to_strings(opts)
113
+ v = v.key_symbols_to_strings(options)
95
114
  when Array
96
115
  # noinspection RubyResolve
97
- v = v.collect { |a| (a.kind_of? Hash) ? a.key_symbols_to_strings(opts) : Marshal.load(Marshal.dump(a)) }
116
+ v = v.collect { |a| (a.kind_of? Hash) ? a.key_symbols_to_strings(options) : Marshal.load(Marshal.dump(a)) }
98
117
  else
99
118
  # noinspection RubyResolve
100
119
  v = Marshal.load(Marshal.dump(v))
@@ -1,5 +1,7 @@
1
+ # Extension class
1
2
  module Kernel
2
3
 
4
+ # Debugging aid: extract the name of the argument of the last caller
3
5
  def extract_argstring_from(name, call_stack)
4
6
  file, line_number = call_stack.first.match(/^(.+):(\d+)/).captures
5
7
  line = File.readlines(file)[line_number.to_i - 1].strip
@@ -8,6 +10,13 @@ module Kernel
8
10
  argstring
9
11
  end
10
12
 
13
+ # Debugging aid: print "<name> : <value>"
14
+ #
15
+ # Example:
16
+ # x = 'abc'
17
+ # dputs x
18
+ # # => x : 'abc'
19
+ #
11
20
  def dputs(value)
12
21
  name = extract_argstring_from :dputs, caller
13
22
  puts "#{name} : '#{value}'"
@@ -1,9 +1,14 @@
1
+ require 'backports/rails/string'
2
+
3
+ # Extension class
1
4
  class String
2
5
 
6
+ # Check if string is empty
3
7
  def blank?
4
8
  self == ''
5
9
  end unless method_defined? :blank?
6
10
 
11
+ # Create sortable object from string. Supports better natural sorting.
7
12
  def sort_form
8
13
  result = []
9
14
  matcher = /^(\D*)(\d*)(.*)$/
@@ -19,30 +24,27 @@ class String
19
24
  result
20
25
  end unless method_defined? :sort_form
21
26
 
22
- def underscore
23
- self.gsub(/::/, '/').
24
- gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
25
- gsub(/([a-z\d])([A-Z])/, '\1_\2').
26
- tr('-', '_').
27
- downcase
28
- end unless method_defined? :underscore
29
-
27
+ # Quote string for command-line use.
30
28
  def quote
31
29
  '\"' + self.gsub(/"/) { |s| '\\' + s[0] } + '\"'
32
30
  end unless method_defined? :quote
33
31
 
32
+ # Escape string for use in Regular Expressions
34
33
  def escape_for_regexp
35
34
  self.gsub(/[\.\+\*\(\)\{\}\|\/\\\^\$"']/) { |s| '\\' + s[0].to_s }
36
35
  end
37
36
 
37
+ # Escape double quotes for usage in code strings.
38
38
  def escape_for_string
39
39
  self.gsub(/"/) { |s| '\\' + s[0].to_s }
40
40
  end
41
41
 
42
+ # Escape double quotes for usage in passing through scripts
42
43
  def escape_for_cmd
43
44
  self.gsub(/"/) { |s| '\\\\\\' + s[0].to_s }
44
45
  end
45
46
 
47
+ # Escape single quotes for usage in SQL statements
46
48
  def escape_for_sql
47
49
  self.gsub(/'/) { |s| ($` == '' || $' == '' ? '' : '\'') + s[0].to_s }
48
50
  end
@@ -51,19 +53,23 @@ class String
51
53
  self.gsub /^(\d+|error|float|string);\\?#/, ''
52
54
  end
53
55
 
56
+ # Convert whitespace into underscores
54
57
  def remove_whitespace
55
58
  self.gsub(/\s/, '_')
56
59
  end
57
60
 
61
+ # Escape all not-printabe characters in hex format
58
62
  def encode_visual(regex = nil)
59
63
  regex ||= /\W/
60
64
  self.gsub(regex) { |c| '_x' + '%04x' % c.unpack('U')[0] + '_'}
61
65
  end unless method_defined? :encode_visual
62
66
 
67
+ # Convert all not-printable characters encoded in hex format back to original
63
68
  def decode_visual
64
69
  self.gsub(/_x([0-9a-f]{4})_/i) { [$1.to_i(16)].pack('U') }
65
70
  end unless method_defined? :decode_visual
66
71
 
72
+ # Align a multi-line string to the left by removing as much spaces from the left as possible.
67
73
  def align_left
68
74
  string = dup
69
75
  relevant_lines = string.split(/\r\n|\r|\n/).select { |line| line.size > 0 }
@@ -78,7 +84,10 @@ class String
78
84
 
79
85
  end
80
86
 
87
+ # Extension class
81
88
  class NilClass
89
+
90
+ # Allow nil.blank? so that blank? can be applied without errors.
82
91
  def blank?
83
92
  true
84
93
  end
@@ -7,64 +7,115 @@ require 'libis/tools/extend/string'
7
7
  module Libis
8
8
  module Tools
9
9
 
10
- # The Logger module adds logging functionality to any class.
10
+ # This module adds logging functionality to any class.
11
11
  #
12
12
  # Just include the ::Libis::Tools::Logger module and the methods debug, info, warn, error and fatal will be
13
13
  # available to the class instance. Each method takes a message argument and optional extra parameters.
14
14
  #
15
- # The methods all call the {#message} method with the logging level as first argument and the supplied arguments
16
- # appended.
15
+ # It is possible to overwrite the {#logger} method with your own implementation to use
16
+ # a different logger for your class.
17
+ #
18
+ # The methods all call the {#message} method with the logging level as first argument
19
+ # and the supplied arguments appended.
20
+ #
21
+ # Example:
22
+ #
23
+ # require 'libis/tools/logger'
24
+ # class TestLogger
25
+ # include ::Libis::Tools::Logger
26
+ # attr_accessor :options, name
27
+ # end
28
+ # tl = TestLogger.new
29
+ # tl.debug 'message'
30
+ # tl.warn 'message'
31
+ # tl.error 'huge error: [%d] %s', 1000, 'Exit'
32
+ # tl.info 'Running application: %s', t.class.name
33
+ #
34
+ # produces:
35
+ # D, [...] DEBUG : message
36
+ # W, [...] WARN : message
37
+ # E, [...] ERROR : huge error: [1000] Exit
38
+ # I, [...] INFO : Running application TestLogger
39
+ #
17
40
  module Logger
18
41
 
19
- def self.included(klass)
20
- klass.class_eval do
21
-
22
- def debug(msg, *args)
23
- return if (self.options[:quiet] rescue false)
24
- message ::Logger::DEBUG, msg, *args
25
- end
42
+ # Get the logger instance
43
+ #
44
+ # Default implementation is to get the root logger from the Config, but can be overwritten for sub-loggers.
45
+ # @!method(logger)
46
+ def logger
47
+ Config.logger
48
+ end
26
49
 
27
- def info(msg, *args)
28
- return if (self.options[:quiet] rescue false)
29
- message ::Logger::INFO, msg, *args
30
- end
50
+ # Send a debug message to the logger.
51
+ #
52
+ # If the optional extra parameters are supplied, the first parameter will be interpreted as a format
53
+ # specification. It's up to the caller to make sure the format specification complies with the number and
54
+ # types of the extra arguments. If the format substitution fails, the message will be printed as:
55
+ # '<msg> - [<args>]'.
56
+ #
57
+ # @param [String] msg the message.
58
+ # @param [Array] args optional extra arguments.
59
+ # @!method(debug(msg, *args))
60
+ def debug(msg, *args)
61
+ message :DEBUG, msg, *args
62
+ end
31
63
 
32
- def warn(msg, *args)
33
- return if (self.options[:quiet] rescue false)
34
- message ::Logger::WARN, msg, *args
35
- end
64
+ # Send an info message to the logger.
65
+ #
66
+ # (see #debug)
67
+ # @param (see #debug)
68
+ # @!method(info(msg, *args))
69
+ def info(msg, *args)
70
+ message :INFO, msg, *args
71
+ end
36
72
 
37
- def error(msg, *args)
38
- message ::Logger::ERROR, msg, *args
39
- end
73
+ # Send a warning message to the logger.
74
+ #
75
+ # (see #debug)
76
+ # @param (see #debug)
77
+ # @!method(warn(msg, *args))
78
+ def warn(msg, *args)
79
+ message :WARN, msg, *args
80
+ end
40
81
 
41
- def fatal(msg, *args)
42
- message ::Logger::FATAL, msg, *args
43
- end
82
+ # Send an error message to the logger.
83
+ #
84
+ # (see #debug)
85
+ # @param (see #debug)
86
+ # @!method(error(msg, *args))
87
+ def error(msg, *args)
88
+ message :ERROR, msg, *args
89
+ end
44
90
 
45
- # The method that performs the code logging action.
46
- #
47
- # If extra arguments are supplied, the message string is expected to be a format specification string and the
48
- # extra arguments will be applied to it.
49
- #
50
- # This default message method implementation uses the logger of ::Libis::Tools::Config. If an 'appname'
51
- # parameter is defined in the Config object, it will be used as program name by the logger, otherwise the
52
- # class name is taken.
53
- #
54
- # @param [{::Logger::Severity}] severity
55
- # @param [String] msg message string
56
- # @param [Object] args optional list of extra arguments
57
- def message(severity, msg, *args)
58
- message_text = (msg % args rescue ((msg + ' - %s') % args.to_s))
59
- appname = Config.appname
60
- appname = self.name if self.respond_to? :name
61
- appname = self.class.name if appname.blank?
62
- Config.logger.add(severity, message_text, appname)
63
- end
91
+ # Send a fatal message to the logger.
92
+ #
93
+ # (see #debug)
94
+ # @param (see #debug)
95
+ # @!method(fatal(msg, *args))
96
+ def fatal(msg, *args)
97
+ message :FATAL, msg, *args
98
+ end
64
99
 
65
- end
100
+ # The method that performs the code logging action.
101
+ #
102
+ # If extra arguments are supplied, the message string is expected to be a format specification string and the
103
+ # extra arguments will be applied to it.
104
+ #
105
+ # This default message method implementation uses the logger of ::Libis::Tools::Config. If an 'appname'
106
+ # parameter is defined in the Config object, it will be used as program name by the logger, otherwise the
107
+ # class name is taken.
108
+ #
109
+ # @param [{::Logger::Severity}] severity
110
+ # @param [String] msg message string
111
+ # @param [Object] args optional list of extra arguments
112
+ # @!method(message(severity, msg, *args))
113
+ def message(severity, msg, *args)
114
+ message_text = (msg % args rescue ((msg + ' - %s') % args.to_s))
115
+ self.logger.add(::Logging.level_num(severity), message_text)
66
116
  end
67
117
 
118
+
68
119
  end
69
120
 
70
121
  end
@@ -1,11 +1,15 @@
1
1
  module Libis
2
2
  module Tools
3
+
4
+ # This module groups several metadata formats. Note that all metadata related classes will move into a Gem of their
5
+ # own in the future.
3
6
  module Metadata
4
7
 
5
8
  autoload :MarcRecord, 'libis/tools/metadata/marc_record'
6
9
  autoload :Marc21Record, 'libis/tools/metadata/marc21_record'
7
10
  autoload :DublinCoreRecord, 'libis/tools/metadata/dublin_core_record'
8
11
 
12
+ # Mappers implementations for converting MARC records to Dublin Core.
9
13
  module Mappers
10
14
 
11
15
  autoload :Kuleuven, 'libis/tools/metadata/mappers/kuleuven'
@@ -17,4 +21,4 @@ module Libis
17
21
  end
18
22
  end
19
23
 
20
- require_relative 'metadata/parsers'
24
+ require_relative 'metadata/parsers'