libis-tools 0.9.20 → 0.9.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +36 -233
- data/Rakefile +5 -0
- data/lib/libis/tools.rb +1 -0
- data/lib/libis/tools/assert.rb +11 -0
- data/lib/libis/tools/checksum.rb +22 -5
- data/lib/libis/tools/command.rb +24 -3
- data/lib/libis/tools/config.rb +61 -33
- data/lib/libis/tools/config_file.rb +0 -1
- data/lib/libis/tools/deep_struct.rb +10 -2
- data/lib/libis/tools/extend/empty.rb +2 -2
- data/lib/libis/tools/extend/hash.rb +37 -18
- data/lib/libis/tools/extend/kernel.rb +9 -0
- data/lib/libis/tools/extend/string.rb +17 -8
- data/lib/libis/tools/logger.rb +95 -44
- data/lib/libis/tools/metadata.rb +5 -1
- data/lib/libis/tools/metadata/dublin_core_record.rb +22 -4
- data/lib/libis/tools/metadata/field_format.rb +49 -9
- data/lib/libis/tools/metadata/fix_field.rb +5 -0
- data/lib/libis/tools/metadata/mapper.rb +2 -1
- data/lib/libis/tools/metadata/mappers/flandrica.rb +8 -1
- data/lib/libis/tools/metadata/mappers/kuleuven.rb +6 -2
- data/lib/libis/tools/metadata/marc21_record.rb +1 -0
- data/lib/libis/tools/metadata/marc_record.rb +31 -12
- data/lib/libis/tools/metadata/parser/basic_parser.rb +2 -0
- data/lib/libis/tools/metadata/parser/dublin_core_parser.rb +2 -1
- data/lib/libis/tools/metadata/parser/marc21_parser.rb +2 -1
- data/lib/libis/tools/metadata/parser/marc_format_parser.rb +2 -1
- data/lib/libis/tools/metadata/parser/marc_rules.rb +2 -1
- data/lib/libis/tools/metadata/parser/marc_select_parser.rb +2 -1
- data/lib/libis/tools/metadata/parser/patch.rb +1 -0
- data/lib/libis/tools/metadata/parser/subfield_criteria_parser.rb +2 -1
- data/lib/libis/tools/metadata/sharepoint_mapping.rb +1 -0
- data/lib/libis/tools/metadata/sharepoint_record.rb +2 -0
- data/lib/libis/tools/metadata/var_field.rb +8 -0
- data/lib/libis/tools/mets_dnx.rb +61 -0
- data/lib/libis/tools/mets_file.rb +87 -604
- data/lib/libis/tools/mets_objects.rb +534 -0
- data/lib/libis/tools/parameter.rb +144 -21
- data/lib/libis/tools/thread_safe.rb +31 -0
- data/lib/libis/tools/version.rb +1 -1
- data/lib/libis/tools/xml_document.rb +18 -24
- data/libis-tools.gemspec +6 -2
- data/spec/config_spec.rb +3 -4
- data/spec/logger_spec.rb +13 -30
- data/spec/mets_file_spec.rb +17 -17
- metadata +53 -7
@@ -4,10 +4,18 @@ require 'recursive-open-struct'
|
|
4
4
|
module Libis
|
5
5
|
module Tools
|
6
6
|
|
7
|
-
# A class that derives from OpenStruct through the RecursiveOpenStruct.
|
8
|
-
#
|
7
|
+
# A class that derives from OpenStruct through the RecursiveOpenStruct.
|
8
|
+
# By wrapping a Hash recursively, it allows for easy access to the content by method names.
|
9
|
+
# A RecursiveOpenStruct is derived from stdlib's OpenStruct, but can be made recursive.
|
10
|
+
# DeepStruct enforces this behaviour and adds a clear! method.
|
9
11
|
class DeepStruct < RecursiveOpenStruct
|
10
12
|
|
13
|
+
# Create a new DeepStruct from a Hash and configure the behaviour.
|
14
|
+
#
|
15
|
+
# @param [Hash] hash the initial data structure.
|
16
|
+
# @param [Hash] opts optional configuration options:
|
17
|
+
# * recurse_over_arrays: also wrap the Hashes that are enbedded in Arrays. Default: true.
|
18
|
+
# * preserver_original_keys: creating a Hash from the wrapper preserves symbols and strings as keys. Default: true.
|
11
19
|
def initialize(hash = {}, opts = {})
|
12
20
|
hash = {} unless hash
|
13
21
|
opts = {} unless opts
|
@@ -1,11 +1,14 @@
|
|
1
1
|
require 'backports/rails/hash'
|
2
2
|
|
3
|
+
# Extension class for Hash
|
3
4
|
class Hash
|
4
5
|
|
6
|
+
# Removes all hash entries for which value.empty? is true
|
5
7
|
def cleanup
|
6
8
|
self.delete_if { |_,v| v.nil? || (v.respond_to?(:empty?) ? v.empty? : false) }
|
7
9
|
end unless method_defined? :cleanup
|
8
10
|
|
11
|
+
# Removes all hash entries for which value.empty? is true. Performed recursively.
|
9
12
|
def recursive_cleanup
|
10
13
|
delete_proc = Proc.new do |_, v|
|
11
14
|
v.delete_if(&delete_proc) if v.kind_of?(Hash)
|
@@ -14,6 +17,7 @@ class Hash
|
|
14
17
|
self.delete_if &delete_proc
|
15
18
|
end unless method_defined? :recursive_cleanup
|
16
19
|
|
20
|
+
# Merges two hashes, but does so recursively.
|
17
21
|
def recursive_merge(other_hash)
|
18
22
|
self.merge(other_hash) do |_, old_val, new_val|
|
19
23
|
if old_val.is_a? Hash
|
@@ -24,6 +28,7 @@ class Hash
|
|
24
28
|
end
|
25
29
|
end unless method_defined? :recursive_merge
|
26
30
|
|
31
|
+
# Merges two hashes in-place, but does so recursively.
|
27
32
|
def recursive_merge!(other_hash)
|
28
33
|
self.merge!(other_hash) do |_, old_val, new_val|
|
29
34
|
if old_val.is_a? Hash
|
@@ -34,30 +39,38 @@ class Hash
|
|
34
39
|
end
|
35
40
|
end unless method_defined? :recursive_merge!
|
36
41
|
|
37
|
-
|
38
|
-
|
42
|
+
# Convert all keys to symbols. In-place operation.
|
43
|
+
# @param (see #key_strings_to_symbols)
|
44
|
+
def key_strings_to_symbols!(options = {})
|
45
|
+
self.replace self.key_strings_to_symbols options
|
39
46
|
end unless method_defined? :key_strings_to_symbols!
|
40
47
|
|
41
|
-
|
42
|
-
|
48
|
+
# Return new Hash with all keys converted to symbols.
|
49
|
+
# @param [Hash] opts valid options are:
|
50
|
+
# * recursive : perform operation recursively
|
51
|
+
# * upcase : convert all keys to upper case
|
52
|
+
# * downcase : convert all keys to lower case
|
53
|
+
# all options are false by default
|
54
|
+
def key_strings_to_symbols(options = {})
|
55
|
+
options = {resursive: false, upcase: false, downcase: false}.merge options
|
43
56
|
|
44
57
|
r = Hash.new
|
45
58
|
self.each_pair do |k,v|
|
46
59
|
|
47
60
|
k = k.to_s if k.kind_of? Symbol
|
48
61
|
if k.kind_of? String
|
49
|
-
k = k.downcase if
|
50
|
-
k = k.upcase if
|
62
|
+
k = k.downcase if options[:downcase]
|
63
|
+
k = k.upcase if options[:upcase]
|
51
64
|
k = k.to_sym
|
52
65
|
end
|
53
66
|
|
54
|
-
if
|
67
|
+
if options[:recursive]
|
55
68
|
case v
|
56
69
|
when Hash
|
57
|
-
v = v.key_strings_to_symbols
|
70
|
+
v = v.key_strings_to_symbols options
|
58
71
|
when Array
|
59
72
|
# noinspection RubyResolve
|
60
|
-
v = v.collect { |a| (a.kind_of? Hash) ? a.key_strings_to_symbols(
|
73
|
+
v = v.collect { |a| (a.kind_of? Hash) ? a.key_strings_to_symbols(options) : Marshal.load(Marshal.dump(a)) }
|
61
74
|
else
|
62
75
|
# noinspection RubyResolve
|
63
76
|
v = Marshal.load(Marshal.dump(v))
|
@@ -71,12 +84,18 @@ class Hash
|
|
71
84
|
r
|
72
85
|
end unless method_defined? :key_strings_to_symbols
|
73
86
|
|
74
|
-
|
75
|
-
|
87
|
+
# Convert all keys to strings. In-place operation.
|
88
|
+
# (@see #key_symbols_to_strings)
|
89
|
+
# @param (see #key_symbols_to_strings)
|
90
|
+
def key_symbols_to_strings!(options = {})
|
91
|
+
self.replace self.key_symbols_to_strings options
|
76
92
|
end unless method_defined? :key_symbols_to_strings!
|
77
93
|
|
78
|
-
|
79
|
-
|
94
|
+
# Return new Hash with all keys converted to strings.
|
95
|
+
# (see #key_strings_to_symbols)
|
96
|
+
# @param (see #key_strings_to_symbols)
|
97
|
+
def key_symbols_to_strings(options = {})
|
98
|
+
options = {resursive: false, upcase: false, downcase: false}.merge options
|
80
99
|
|
81
100
|
r = Hash.new
|
82
101
|
self.each_pair do |k,v|
|
@@ -84,17 +103,17 @@ class Hash
|
|
84
103
|
k = k.to_sym if k.kind_of? String
|
85
104
|
if k.kind_of? Symbol
|
86
105
|
k = k.to_s
|
87
|
-
k = k.downcase if
|
88
|
-
k = k.upcase if
|
106
|
+
k = k.downcase if options[:downcase]
|
107
|
+
k = k.upcase if options[:upcase]
|
89
108
|
end
|
90
109
|
|
91
|
-
if
|
110
|
+
if options[:recursive]
|
92
111
|
case v
|
93
112
|
when Hash
|
94
|
-
v = v.key_symbols_to_strings(
|
113
|
+
v = v.key_symbols_to_strings(options)
|
95
114
|
when Array
|
96
115
|
# noinspection RubyResolve
|
97
|
-
v = v.collect { |a| (a.kind_of? Hash) ? a.key_symbols_to_strings(
|
116
|
+
v = v.collect { |a| (a.kind_of? Hash) ? a.key_symbols_to_strings(options) : Marshal.load(Marshal.dump(a)) }
|
98
117
|
else
|
99
118
|
# noinspection RubyResolve
|
100
119
|
v = Marshal.load(Marshal.dump(v))
|
@@ -1,5 +1,7 @@
|
|
1
|
+
# Extension class
|
1
2
|
module Kernel
|
2
3
|
|
4
|
+
# Debugging aid: extract the name of the argument of the last caller
|
3
5
|
def extract_argstring_from(name, call_stack)
|
4
6
|
file, line_number = call_stack.first.match(/^(.+):(\d+)/).captures
|
5
7
|
line = File.readlines(file)[line_number.to_i - 1].strip
|
@@ -8,6 +10,13 @@ module Kernel
|
|
8
10
|
argstring
|
9
11
|
end
|
10
12
|
|
13
|
+
# Debugging aid: print "<name> : <value>"
|
14
|
+
#
|
15
|
+
# Example:
|
16
|
+
# x = 'abc'
|
17
|
+
# dputs x
|
18
|
+
# # => x : 'abc'
|
19
|
+
#
|
11
20
|
def dputs(value)
|
12
21
|
name = extract_argstring_from :dputs, caller
|
13
22
|
puts "#{name} : '#{value}'"
|
@@ -1,9 +1,14 @@
|
|
1
|
+
require 'backports/rails/string'
|
2
|
+
|
3
|
+
# Extension class
|
1
4
|
class String
|
2
5
|
|
6
|
+
# Check if string is empty
|
3
7
|
def blank?
|
4
8
|
self == ''
|
5
9
|
end unless method_defined? :blank?
|
6
10
|
|
11
|
+
# Create sortable object from string. Supports better natural sorting.
|
7
12
|
def sort_form
|
8
13
|
result = []
|
9
14
|
matcher = /^(\D*)(\d*)(.*)$/
|
@@ -19,30 +24,27 @@ class String
|
|
19
24
|
result
|
20
25
|
end unless method_defined? :sort_form
|
21
26
|
|
22
|
-
|
23
|
-
self.gsub(/::/, '/').
|
24
|
-
gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').
|
25
|
-
gsub(/([a-z\d])([A-Z])/, '\1_\2').
|
26
|
-
tr('-', '_').
|
27
|
-
downcase
|
28
|
-
end unless method_defined? :underscore
|
29
|
-
|
27
|
+
# Quote string for command-line use.
|
30
28
|
def quote
|
31
29
|
'\"' + self.gsub(/"/) { |s| '\\' + s[0] } + '\"'
|
32
30
|
end unless method_defined? :quote
|
33
31
|
|
32
|
+
# Escape string for use in Regular Expressions
|
34
33
|
def escape_for_regexp
|
35
34
|
self.gsub(/[\.\+\*\(\)\{\}\|\/\\\^\$"']/) { |s| '\\' + s[0].to_s }
|
36
35
|
end
|
37
36
|
|
37
|
+
# Escape double quotes for usage in code strings.
|
38
38
|
def escape_for_string
|
39
39
|
self.gsub(/"/) { |s| '\\' + s[0].to_s }
|
40
40
|
end
|
41
41
|
|
42
|
+
# Escape double quotes for usage in passing through scripts
|
42
43
|
def escape_for_cmd
|
43
44
|
self.gsub(/"/) { |s| '\\\\\\' + s[0].to_s }
|
44
45
|
end
|
45
46
|
|
47
|
+
# Escape single quotes for usage in SQL statements
|
46
48
|
def escape_for_sql
|
47
49
|
self.gsub(/'/) { |s| ($` == '' || $' == '' ? '' : '\'') + s[0].to_s }
|
48
50
|
end
|
@@ -51,19 +53,23 @@ class String
|
|
51
53
|
self.gsub /^(\d+|error|float|string);\\?#/, ''
|
52
54
|
end
|
53
55
|
|
56
|
+
# Convert whitespace into underscores
|
54
57
|
def remove_whitespace
|
55
58
|
self.gsub(/\s/, '_')
|
56
59
|
end
|
57
60
|
|
61
|
+
# Escape all not-printabe characters in hex format
|
58
62
|
def encode_visual(regex = nil)
|
59
63
|
regex ||= /\W/
|
60
64
|
self.gsub(regex) { |c| '_x' + '%04x' % c.unpack('U')[0] + '_'}
|
61
65
|
end unless method_defined? :encode_visual
|
62
66
|
|
67
|
+
# Convert all not-printable characters encoded in hex format back to original
|
63
68
|
def decode_visual
|
64
69
|
self.gsub(/_x([0-9a-f]{4})_/i) { [$1.to_i(16)].pack('U') }
|
65
70
|
end unless method_defined? :decode_visual
|
66
71
|
|
72
|
+
# Align a multi-line string to the left by removing as much spaces from the left as possible.
|
67
73
|
def align_left
|
68
74
|
string = dup
|
69
75
|
relevant_lines = string.split(/\r\n|\r|\n/).select { |line| line.size > 0 }
|
@@ -78,7 +84,10 @@ class String
|
|
78
84
|
|
79
85
|
end
|
80
86
|
|
87
|
+
# Extension class
|
81
88
|
class NilClass
|
89
|
+
|
90
|
+
# Allow nil.blank? so that blank? can be applied without errors.
|
82
91
|
def blank?
|
83
92
|
true
|
84
93
|
end
|
data/lib/libis/tools/logger.rb
CHANGED
@@ -7,64 +7,115 @@ require 'libis/tools/extend/string'
|
|
7
7
|
module Libis
|
8
8
|
module Tools
|
9
9
|
|
10
|
-
#
|
10
|
+
# This module adds logging functionality to any class.
|
11
11
|
#
|
12
12
|
# Just include the ::Libis::Tools::Logger module and the methods debug, info, warn, error and fatal will be
|
13
13
|
# available to the class instance. Each method takes a message argument and optional extra parameters.
|
14
14
|
#
|
15
|
-
#
|
16
|
-
#
|
15
|
+
# It is possible to overwrite the {#logger} method with your own implementation to use
|
16
|
+
# a different logger for your class.
|
17
|
+
#
|
18
|
+
# The methods all call the {#message} method with the logging level as first argument
|
19
|
+
# and the supplied arguments appended.
|
20
|
+
#
|
21
|
+
# Example:
|
22
|
+
#
|
23
|
+
# require 'libis/tools/logger'
|
24
|
+
# class TestLogger
|
25
|
+
# include ::Libis::Tools::Logger
|
26
|
+
# attr_accessor :options, name
|
27
|
+
# end
|
28
|
+
# tl = TestLogger.new
|
29
|
+
# tl.debug 'message'
|
30
|
+
# tl.warn 'message'
|
31
|
+
# tl.error 'huge error: [%d] %s', 1000, 'Exit'
|
32
|
+
# tl.info 'Running application: %s', t.class.name
|
33
|
+
#
|
34
|
+
# produces:
|
35
|
+
# D, [...] DEBUG : message
|
36
|
+
# W, [...] WARN : message
|
37
|
+
# E, [...] ERROR : huge error: [1000] Exit
|
38
|
+
# I, [...] INFO : Running application TestLogger
|
39
|
+
#
|
17
40
|
module Logger
|
18
41
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
42
|
+
# Get the logger instance
|
43
|
+
#
|
44
|
+
# Default implementation is to get the root logger from the Config, but can be overwritten for sub-loggers.
|
45
|
+
# @!method(logger)
|
46
|
+
def logger
|
47
|
+
Config.logger
|
48
|
+
end
|
26
49
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
50
|
+
# Send a debug message to the logger.
|
51
|
+
#
|
52
|
+
# If the optional extra parameters are supplied, the first parameter will be interpreted as a format
|
53
|
+
# specification. It's up to the caller to make sure the format specification complies with the number and
|
54
|
+
# types of the extra arguments. If the format substitution fails, the message will be printed as:
|
55
|
+
# '<msg> - [<args>]'.
|
56
|
+
#
|
57
|
+
# @param [String] msg the message.
|
58
|
+
# @param [Array] args optional extra arguments.
|
59
|
+
# @!method(debug(msg, *args))
|
60
|
+
def debug(msg, *args)
|
61
|
+
message :DEBUG, msg, *args
|
62
|
+
end
|
31
63
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
64
|
+
# Send an info message to the logger.
|
65
|
+
#
|
66
|
+
# (see #debug)
|
67
|
+
# @param (see #debug)
|
68
|
+
# @!method(info(msg, *args))
|
69
|
+
def info(msg, *args)
|
70
|
+
message :INFO, msg, *args
|
71
|
+
end
|
36
72
|
|
37
|
-
|
38
|
-
|
39
|
-
|
73
|
+
# Send a warning message to the logger.
|
74
|
+
#
|
75
|
+
# (see #debug)
|
76
|
+
# @param (see #debug)
|
77
|
+
# @!method(warn(msg, *args))
|
78
|
+
def warn(msg, *args)
|
79
|
+
message :WARN, msg, *args
|
80
|
+
end
|
40
81
|
|
41
|
-
|
42
|
-
|
43
|
-
|
82
|
+
# Send an error message to the logger.
|
83
|
+
#
|
84
|
+
# (see #debug)
|
85
|
+
# @param (see #debug)
|
86
|
+
# @!method(error(msg, *args))
|
87
|
+
def error(msg, *args)
|
88
|
+
message :ERROR, msg, *args
|
89
|
+
end
|
44
90
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
#
|
54
|
-
# @param [{::Logger::Severity}] severity
|
55
|
-
# @param [String] msg message string
|
56
|
-
# @param [Object] args optional list of extra arguments
|
57
|
-
def message(severity, msg, *args)
|
58
|
-
message_text = (msg % args rescue ((msg + ' - %s') % args.to_s))
|
59
|
-
appname = Config.appname
|
60
|
-
appname = self.name if self.respond_to? :name
|
61
|
-
appname = self.class.name if appname.blank?
|
62
|
-
Config.logger.add(severity, message_text, appname)
|
63
|
-
end
|
91
|
+
# Send a fatal message to the logger.
|
92
|
+
#
|
93
|
+
# (see #debug)
|
94
|
+
# @param (see #debug)
|
95
|
+
# @!method(fatal(msg, *args))
|
96
|
+
def fatal(msg, *args)
|
97
|
+
message :FATAL, msg, *args
|
98
|
+
end
|
64
99
|
|
65
|
-
|
100
|
+
# The method that performs the code logging action.
|
101
|
+
#
|
102
|
+
# If extra arguments are supplied, the message string is expected to be a format specification string and the
|
103
|
+
# extra arguments will be applied to it.
|
104
|
+
#
|
105
|
+
# This default message method implementation uses the logger of ::Libis::Tools::Config. If an 'appname'
|
106
|
+
# parameter is defined in the Config object, it will be used as program name by the logger, otherwise the
|
107
|
+
# class name is taken.
|
108
|
+
#
|
109
|
+
# @param [{::Logger::Severity}] severity
|
110
|
+
# @param [String] msg message string
|
111
|
+
# @param [Object] args optional list of extra arguments
|
112
|
+
# @!method(message(severity, msg, *args))
|
113
|
+
def message(severity, msg, *args)
|
114
|
+
message_text = (msg % args rescue ((msg + ' - %s') % args.to_s))
|
115
|
+
self.logger.add(::Logging.level_num(severity), message_text)
|
66
116
|
end
|
67
117
|
|
118
|
+
|
68
119
|
end
|
69
120
|
|
70
121
|
end
|
data/lib/libis/tools/metadata.rb
CHANGED
@@ -1,11 +1,15 @@
|
|
1
1
|
module Libis
|
2
2
|
module Tools
|
3
|
+
|
4
|
+
# This module groups several metadata formats. Note that all metadata related classes will move into a Gem of their
|
5
|
+
# own in the future.
|
3
6
|
module Metadata
|
4
7
|
|
5
8
|
autoload :MarcRecord, 'libis/tools/metadata/marc_record'
|
6
9
|
autoload :Marc21Record, 'libis/tools/metadata/marc21_record'
|
7
10
|
autoload :DublinCoreRecord, 'libis/tools/metadata/dublin_core_record'
|
8
11
|
|
12
|
+
# Mappers implementations for converting MARC records to Dublin Core.
|
9
13
|
module Mappers
|
10
14
|
|
11
15
|
autoload :Kuleuven, 'libis/tools/metadata/mappers/kuleuven'
|
@@ -17,4 +21,4 @@ module Libis
|
|
17
21
|
end
|
18
22
|
end
|
19
23
|
|
20
|
-
require_relative 'metadata/parsers'
|
24
|
+
require_relative 'metadata/parsers'
|