libis-tools 1.0.5-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (71) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +2 -0
  3. data/.gitignore +16 -0
  4. data/.rspec +2 -0
  5. data/.travis.yml +40 -0
  6. data/Gemfile +7 -0
  7. data/README.md +202 -0
  8. data/Rakefile +11 -0
  9. data/bin/libis_tool +5 -0
  10. data/lib/libis-tools.rb +1 -0
  11. data/lib/libis/tools.rb +25 -0
  12. data/lib/libis/tools/assert.rb +52 -0
  13. data/lib/libis/tools/checksum.rb +106 -0
  14. data/lib/libis/tools/cli/cli_helper.rb +189 -0
  15. data/lib/libis/tools/cli/reorg.rb +416 -0
  16. data/lib/libis/tools/command.rb +133 -0
  17. data/lib/libis/tools/command_line.rb +23 -0
  18. data/lib/libis/tools/config.rb +147 -0
  19. data/lib/libis/tools/config_file.rb +85 -0
  20. data/lib/libis/tools/csv.rb +38 -0
  21. data/lib/libis/tools/deep_struct.rb +71 -0
  22. data/lib/libis/tools/extend/array.rb +16 -0
  23. data/lib/libis/tools/extend/empty.rb +7 -0
  24. data/lib/libis/tools/extend/hash.rb +147 -0
  25. data/lib/libis/tools/extend/kernel.rb +25 -0
  26. data/lib/libis/tools/extend/ostruct.rb +3 -0
  27. data/lib/libis/tools/extend/roo.rb +91 -0
  28. data/lib/libis/tools/extend/string.rb +94 -0
  29. data/lib/libis/tools/extend/struct.rb +29 -0
  30. data/lib/libis/tools/extend/symbol.rb +8 -0
  31. data/lib/libis/tools/logger.rb +130 -0
  32. data/lib/libis/tools/mets_dnx.rb +61 -0
  33. data/lib/libis/tools/mets_file.rb +504 -0
  34. data/lib/libis/tools/mets_objects.rb +547 -0
  35. data/lib/libis/tools/parameter.rb +372 -0
  36. data/lib/libis/tools/spreadsheet.rb +196 -0
  37. data/lib/libis/tools/temp_file.rb +42 -0
  38. data/lib/libis/tools/thread_safe.rb +31 -0
  39. data/lib/libis/tools/version.rb +5 -0
  40. data/lib/libis/tools/xml_document.rb +583 -0
  41. data/libis-tools.gemspec +55 -0
  42. data/spec/assert_spec.rb +65 -0
  43. data/spec/checksum_spec.rb +68 -0
  44. data/spec/command_spec.rb +90 -0
  45. data/spec/config_file_spec.rb +83 -0
  46. data/spec/config_spec.rb +113 -0
  47. data/spec/csv_spec.rb +159 -0
  48. data/spec/data/test-headers.csv +2 -0
  49. data/spec/data/test-headers.tsv +2 -0
  50. data/spec/data/test-noheaders.csv +1 -0
  51. data/spec/data/test-noheaders.tsv +1 -0
  52. data/spec/data/test.data +9 -0
  53. data/spec/data/test.xlsx +0 -0
  54. data/spec/data/test.xml +8 -0
  55. data/spec/data/test.yml +2 -0
  56. data/spec/data/test_config.yml +15 -0
  57. data/spec/deep_struct_spec.rb +138 -0
  58. data/spec/logger_spec.rb +165 -0
  59. data/spec/mets_file_spec.rb +223 -0
  60. data/spec/parameter_container_spec.rb +152 -0
  61. data/spec/parameter_spec.rb +148 -0
  62. data/spec/spec_helper.rb +29 -0
  63. data/spec/spreadsheet_spec.rb +1820 -0
  64. data/spec/temp_file_spec.rb +76 -0
  65. data/spec/test.xsd +20 -0
  66. data/spec/thread_safe_spec.rb +64 -0
  67. data/spec/xmldocument_spec.rb +421 -0
  68. data/test/test_helper.rb +7 -0
  69. data/test/webservices/test_ca_item_info.rb +59 -0
  70. data/test/webservices/test_ca_search.rb +35 -0
  71. metadata +437 -0
@@ -0,0 +1,16 @@
1
+
2
+ # Extension class for Array
3
+ class Array
4
+
5
+ # Removes all empty entries
6
+ def cleanup
7
+ self.delete_if { |v| v.nil? || (v.respond_to?(:empty?) ? v.empty? : false) }
8
+ end unless method_defined? :cleanup
9
+
10
+ # Removes all empty entries recursively in the array and each Hash in it
11
+ def recursive_cleanup
12
+ each { |v| v.recursive_cleanup if Array === v || Hash === v }
13
+ cleanup
14
+ end unless method_defined? :recursive_cleanup
15
+
16
+ end
@@ -0,0 +1,7 @@
1
+ # Extension for NilClass
2
+ class NilClass
3
+ # Allows nil.empty?
4
+ def empty?
5
+ true
6
+ end
7
+ end
@@ -0,0 +1,147 @@
1
+ require 'backports/rails/hash'
2
+
3
+ # Extension class for Hash
4
+ class Hash
5
+
6
+ # Removes all hash entries for which value.empty? is true
7
+ def cleanup
8
+ self.delete_if { |_,v| v.nil? || (v.respond_to?(:empty?) ? v.empty? : false) }
9
+ end unless method_defined? :cleanup
10
+
11
+ # Removes all hash entries for which value.empty? is true. Performed recursively.
12
+ def recursive_cleanup
13
+ each { |_, v| v.recursive_cleanup if Array === v || Hash === v }
14
+ cleanup
15
+ end unless method_defined? :recursive_cleanup
16
+
17
+ # Merges two hashes, but does so recursively.
18
+ def recursive_merge(other_hash)
19
+ self.merge(other_hash) do |_, old_val, new_val|
20
+ if old_val.is_a? Hash
21
+ old_val.recursive_merge new_val
22
+ else
23
+ new_val
24
+ end
25
+ end
26
+ end unless method_defined? :recursive_merge
27
+
28
+ # Merges two hashes in-place, but does so recursively.
29
+ def recursive_merge!(other_hash)
30
+ self.merge!(other_hash) do |_, old_val, new_val|
31
+ if old_val.is_a? Hash
32
+ old_val.recursive_merge new_val
33
+ else
34
+ new_val
35
+ end
36
+ end
37
+ end unless method_defined? :recursive_merge!
38
+
39
+ # Merges two hashes with priority for the first hash
40
+ def reverse_merge(other_hash)
41
+ self.merge(other_hash) {|_,v, _| v}
42
+ end unless method_defined? :reverse_merge
43
+
44
+ # Merges two hashes in-place with priority for the first hash
45
+ def reverse_merge!(other_hash)
46
+ self.merge!(other_hash) {|_,v, _| v}
47
+ end unless method_defined? :reverse_merge!
48
+
49
+ # Apply other hash values if current value is blank
50
+ def apply_defaults(other_hash)
51
+ self.merge(other_hash) {|_,v, w| v.blank? ? w : v}
52
+ end unless method_defined? :apply_defaults
53
+
54
+ # Apply in-place other hash values if current value is blank
55
+ def apply_defaults!(other_hash)
56
+ self.merge!(other_hash) {|_,v, w| v.blank? ? w : v}
57
+ end unless method_defined? :apply_defaults!
58
+
59
+ # Convert all keys to symbols. In-place operation.
60
+ # @param (see #key_strings_to_symbols)
61
+ def key_strings_to_symbols!(options = {})
62
+ self.replace self.key_strings_to_symbols options
63
+ end unless method_defined? :key_strings_to_symbols!
64
+
65
+ # Return new Hash with all keys converted to symbols.
66
+ # @param [Hash] options valid options are:
67
+ # * recursive : perform operation recursively
68
+ # * upcase : convert all keys to upper case
69
+ # * downcase : convert all keys to lower case
70
+ # all options are false by default
71
+ def key_strings_to_symbols(options = {})
72
+ options = {resursive: false, upcase: false, downcase: false}.merge options
73
+
74
+ r = Hash.new
75
+ self.each_pair do |k,v|
76
+
77
+ k = k.to_s if k.kind_of? Symbol
78
+ if k.kind_of? String
79
+ k = k.downcase if options[:downcase]
80
+ k = k.upcase if options[:upcase]
81
+ k = k.to_sym
82
+ end
83
+
84
+ if options[:recursive]
85
+ case v
86
+ when Hash
87
+ v = v.key_strings_to_symbols options
88
+ when Array
89
+ # noinspection RubyResolve
90
+ v = v.collect { |a| (a.kind_of? Hash) ? a.key_strings_to_symbols(options) : Marshal.load(Marshal.dump(a)) }
91
+ else
92
+ # noinspection RubyResolve
93
+ v = Marshal.load(Marshal.dump(v))
94
+ end
95
+ end
96
+
97
+ r[k] = v
98
+
99
+ end
100
+
101
+ r
102
+ end unless method_defined? :key_strings_to_symbols
103
+
104
+ # Convert all keys to strings. In-place operation.
105
+ # (@see #key_symbols_to_strings)
106
+ # @param (see #key_symbols_to_strings)
107
+ def key_symbols_to_strings!(options = {})
108
+ self.replace self.key_symbols_to_strings options
109
+ end unless method_defined? :key_symbols_to_strings!
110
+
111
+ # Return new Hash with all keys converted to strings.
112
+ # (see #key_strings_to_symbols)
113
+ # @param (see #key_strings_to_symbols)
114
+ def key_symbols_to_strings(options = {})
115
+ options = {resursive: false, upcase: false, downcase: false}.merge options
116
+
117
+ r = Hash.new
118
+ self.each_pair do |k,v|
119
+
120
+ k = k.to_sym if k.kind_of? String
121
+ if k.kind_of? Symbol
122
+ k = k.to_s
123
+ k = k.downcase if options[:downcase]
124
+ k = k.upcase if options[:upcase]
125
+ end
126
+
127
+ if options[:recursive]
128
+ case v
129
+ when Hash
130
+ v = v.key_symbols_to_strings(options)
131
+ when Array
132
+ # noinspection RubyResolve
133
+ v = v.collect { |a| (a.kind_of? Hash) ? a.key_symbols_to_strings(options) : Marshal.load(Marshal.dump(a)) }
134
+ else
135
+ # noinspection RubyResolve
136
+ v = Marshal.load(Marshal.dump(v))
137
+ end
138
+ end
139
+
140
+ r[k] = v
141
+
142
+ end
143
+
144
+ r
145
+ end unless method_defined? :key_symbols_to_strings
146
+
147
+ end
@@ -0,0 +1,25 @@
1
+ # Extension class
2
+ module Kernel
3
+
4
+ # Debugging aid: extract the name of the argument of the last caller
5
+ def extract_argstring_from(name, call_stack)
6
+ file, line_number = call_stack.first.match(/^(.+):(\d+)/).captures
7
+ line = File.readlines(file)[line_number.to_i - 1].strip
8
+ argstring = line[/#{name}\s*\(?(.+?)\)?\s*($|#|\[|\})/, 1]
9
+ raise "unable to extract name for #{name} from #{file} line #{line_number}:\n #{line}" unless argstring
10
+ argstring
11
+ end
12
+
13
+ # Debugging aid: print "<name> : <value>"
14
+ #
15
+ # Example:
16
+ # x = 'abc'
17
+ # dputs x
18
+ # # => x : 'abc'
19
+ #
20
+ def dputs(value)
21
+ name = extract_argstring_from :dputs, caller
22
+ puts "#{name} : '#{value}'"
23
+ end
24
+
25
+ end
@@ -0,0 +1,3 @@
1
+ # encoding: utf-8
2
+ require 'ostruct'
3
+ require 'backports/2.0.0/stdlib/ostruct'
@@ -0,0 +1,91 @@
1
+ require 'roo'
2
+ require 'roo-xls'
3
+ require 'libis/tools/extend/hash'
4
+
5
+ module Roo
6
+ class HeaderRowIncompleteError < Error;
7
+ end
8
+ class Base
9
+
10
+ # changes:
11
+ # - added option :partial_match to allow to use headers that only partially match the query
12
+ # - added option :required_headers to force the result to have at least these columns
13
+ # - allow option :headers to contain an array with header labels that will be forced when no header row is found
14
+ # - improved proper range scanning (first_row->last_row and first_column->last_column)
15
+
16
+ attr_accessor :partial_match
17
+
18
+ def each(options = {})
19
+ return to_enum(:each, options) unless block_given?
20
+
21
+ @partial_match = options.delete(:partial_match) if options.has_key?(:partial_match)
22
+ required_headers = options.delete(:required_headers) if options.has_key?(:required_headers)
23
+
24
+ if options.empty?
25
+ first_row.upto(last_row) do |line|
26
+ yield row(line)
27
+ end
28
+ else
29
+ clean_sheet_if_need(options)
30
+ @headers = search_or_set_header(options)
31
+ if required_headers
32
+ raise Roo::HeaderRowIncompleteError unless headers.keys & required_headers == required_headers
33
+ end
34
+
35
+ header_line.upto(last_row) do |line|
36
+ yield(Hash[headers.map { |k, v| [k, cell(line, v)] }])
37
+ end
38
+ end
39
+ end
40
+
41
+ private
42
+
43
+ def row_with(query)
44
+ line_no = first_row
45
+ each do |row|
46
+ headers = query.map { |q| row.grep(q)[0] }.compact
47
+
48
+ if headers.length == query.length
49
+ @header_line = line_no
50
+ return headers
51
+ elsif line_no > 100
52
+ raise Roo::HeaderRowNotFoundError
53
+ elsif headers.length > 0
54
+ # partial match
55
+ @header_line = line_no
56
+ raise Roo::HeaderRowIncompleteError unless partial_match
57
+ return headers
58
+ end
59
+ line_no += 1
60
+ end
61
+ raise Roo::HeaderRowNotFoundError
62
+ end
63
+
64
+ def search_or_set_header(options)
65
+ force_headers = options.delete(:headers)
66
+ if options[:header_search]
67
+ row_with(options[:header_search])
68
+ elsif [:first_row, true].include?(force_headers)
69
+ @header_line = first_row
70
+ else
71
+ return set_headers(options)
72
+ end
73
+ return Hash[row(header_line).map { |x| [x, header_index(x)] }]
74
+ rescue Roo::HeaderRowNotFoundError => e
75
+ # Not OK unless a list of headers is supplied
76
+ raise e unless force_headers.is_a?(Array)
77
+ # Force the headers in the order they are given, but up to the last column
78
+ @header_line = first_row - 1
79
+ return Hash[force_headers.zip(first_column..last_column)].cleanup
80
+ end
81
+
82
+ def set_headers(hash)
83
+ # try to find header row with all values or give an error
84
+ # then create new hash by indexing strings and keeping integers for header array
85
+ row_with(hash.values)
86
+ positions = Hash[row(header_line).map { |x| [x, header_index(x)] }]
87
+ Hash[positions.map { |k, v| [hash.invert[k] || k, v] }]
88
+ end
89
+
90
+ end
91
+ end
@@ -0,0 +1,94 @@
1
+ require 'backports/rails/string'
2
+
3
+ # Extension class
4
+ class String
5
+
6
+ # Check if string is empty
7
+ def blank?
8
+ self == ''
9
+ end unless method_defined? :blank?
10
+
11
+ # Create sortable object from string. Supports better natural sorting.
12
+ def sort_form
13
+ result = []
14
+ matcher = /^(\D*)(\d*)(.*)$/
15
+ self.split('.').each { |s|
16
+ while !s.empty? and (x = matcher.match s)
17
+ a = x[1].to_s.strip
18
+ b = a.gsub(/[ _]/, '')
19
+ result << [b.downcase, b, a]
20
+ result << x[2].to_i
21
+ s = x[3]
22
+ end
23
+ }
24
+ result
25
+ end unless method_defined? :sort_form
26
+
27
+ # Quote string for command-line use.
28
+ def quote
29
+ '\"' + self.gsub(/"/) { |s| '\\' + s[0] } + '\"'
30
+ end unless method_defined? :quote
31
+
32
+ # Escape string for use in Regular Expressions
33
+ def escape_for_regexp
34
+ self.gsub(/[\.\+\*\(\)\{\}\|\/\\\^\$"']/) { |s| '\\' + s[0].to_s }
35
+ end
36
+
37
+ # Escape double quotes for usage in code strings.
38
+ def escape_for_string
39
+ self.gsub(/"/) { |s| '\\' + s[0].to_s }
40
+ end
41
+
42
+ # Escape double quotes for usage in passing through scripts
43
+ def escape_for_cmd
44
+ self.gsub(/"/) { |s| '\\\\\\' + s[0].to_s }
45
+ end
46
+
47
+ # Escape single quotes for usage in SQL statements
48
+ def escape_for_sql
49
+ self.gsub(/'/) { |s| ($` == '' || $' == '' ? '' : '\'') + s[0].to_s }
50
+ end
51
+
52
+ def dot_net_clean
53
+ self.gsub /^(\d+|error|float|string);\\?#/, ''
54
+ end
55
+
56
+ # Convert whitespace into underscores
57
+ def remove_whitespace
58
+ self.gsub(/\s/, '_')
59
+ end
60
+
61
+ # Escape all not-printabe characters in hex format
62
+ def encode_visual(regex = nil)
63
+ regex ||= /\W/
64
+ self.gsub(regex) { |c| '_x' + '%04x' % c.unpack('U')[0] + '_'}
65
+ end unless method_defined? :encode_visual
66
+
67
+ # Convert all not-printable characters encoded in hex format back to original
68
+ def decode_visual
69
+ self.gsub(/_x([0-9a-f]{4})_/i) { [$1.to_i(16)].pack('U') }
70
+ end unless method_defined? :decode_visual
71
+
72
+ # Align a multi-line string to the left by removing as much spaces from the left as possible.
73
+ def align_left
74
+ string = dup
75
+ relevant_lines = string.split(/\r\n|\r|\n/).select { |line| line.size > 0 }
76
+ indentation_levels = relevant_lines.map do |line|
77
+ match = line.match(/^( +)[^ ]+/)
78
+ match ? match[1].size : 0
79
+ end
80
+ indentation_level = indentation_levels.min
81
+ string.gsub! /^#{' ' * indentation_level}/, '' if indentation_level > 0
82
+ string
83
+ end unless method_defined? :align_left
84
+
85
+ end
86
+
87
+ # Extension class
88
+ class NilClass
89
+
90
+ # Allow nil.blank? so that blank? can be applied without errors.
91
+ def blank?
92
+ true
93
+ end
94
+ end
@@ -0,0 +1,29 @@
1
+ # encoding: utf-8
2
+ require 'json'
3
+ require 'backports/rails/hash'
4
+ require 'backports/2.0.0/struct'
5
+
6
+ class Struct
7
+ # def to_hash
8
+ # members.inject({}) {|h,m| h[m] = send(m); h}
9
+ # end unless method_defined? :to_hash
10
+
11
+ def set(h = {})
12
+ h.symbolize_keys!
13
+ members.each {|m| send("#{m}=", h[m]) if h.key?(m)}
14
+ self
15
+ end unless method_defined? :set
16
+
17
+ def self.from_hash(h)
18
+ h.symbolize_keys!
19
+ members.inject(new) {|o,m| o[m] = h[m] if h.key?(m); o}
20
+ end unless respond_to? :from_hash
21
+
22
+ def to_json
23
+ to_hash.to_json
24
+ end unless method_defined? :to_json
25
+
26
+ def self.from_json(j)
27
+ from_hash(JSON.parse(j))
28
+ end unless respond_to? :from_json
29
+ end
@@ -0,0 +1,8 @@
1
+ # Symbol monkey patch to allow map(&:method) to take arguments. Allows: [2,3].map(&:+.(10)) # => [12,13]
2
+ # See: https://stackoverflow.com/questions/23695653/can-you-supply-arguments-to-the-mapmethod-syntax-in-ruby
3
+ # for more information,
4
+ class Symbol
5
+ def call(*args, &block)
6
+ ->(caller, *rest) { caller.public_send(self, *rest, *args, &block) }
7
+ end
8
+ end
@@ -0,0 +1,130 @@
1
+ # encoding: utf-8
2
+
3
+ require 'backports'
4
+ require 'libis/tools/config'
5
+ require 'libis/tools/extend/string'
6
+
7
+ module Libis
8
+ module Tools
9
+
10
+ # This module adds logging functionality to any class.
11
+ #
12
+ # Just include the ::Libis::Tools::Logger module and the methods debug, info, warn, error and fatal will be
13
+ # available to the class instance. Each method takes a message argument and optional extra parameters.
14
+ #
15
+ # It is possible to overwrite the {#logger} method with your own implementation to use
16
+ # a different logger for your class.
17
+ #
18
+ # The methods all call the {#message} method with the logging level as first argument
19
+ # and the supplied arguments appended.
20
+ #
21
+ # Example:
22
+ #
23
+ # require 'libis/tools/logger'
24
+ # class TestLogger
25
+ # include ::Libis::Tools::Logger
26
+ # attr_accessor :options, name
27
+ # end
28
+ # tl = TestLogger.new
29
+ # tl.debug 'message'
30
+ # tl.warn 'message'
31
+ # tl.error 'huge error: [%d] %s', 1000, 'Exit'
32
+ # tl.info 'Running application: %s', t.class.name
33
+ #
34
+ # produces:
35
+ # D, [...] DEBUG : message
36
+ # W, [...] WARN : message
37
+ # E, [...] ERROR : huge error: [1000] Exit
38
+ # I, [...] INFO : Running application TestLogger
39
+ #
40
+ module Logger
41
+
42
+ # Get the logger instance
43
+ #
44
+ # Default implementation is to get the root logger from the Config, but can be overwritten for sub-loggers.
45
+ # @!method(logger)
46
+ def logger
47
+ ::Libis::Tools::Config.logger
48
+ end
49
+
50
+ def set_application(name = nil)
51
+ name ||= self.class.name
52
+ ::Logging.mdc['Application'] = name.blank? ? '' : " -- #{name}"
53
+ end
54
+
55
+ def set_subject(name = nil)
56
+ ::Logging.mdc['Subject'] = name.blank? ? '' : " - #{name}"
57
+ end
58
+
59
+ # Send a debug message to the logger.
60
+ #
61
+ # If the optional extra parameters are supplied, the first parameter will be interpreted as a format
62
+ # specification. It's up to the caller to make sure the format specification complies with the number and
63
+ # types of the extra arguments. If the format substitution fails, the message will be printed as:
64
+ # '<msg> - [<args>]'.
65
+ #
66
+ # @param [String] msg the message.
67
+ # @param [Array] args optional extra arguments.
68
+ # @!method(debug(msg, *args))
69
+ def debug(msg, *args)
70
+ self.message :DEBUG, msg, *args
71
+ end
72
+
73
+ # Send an info message to the logger.
74
+ #
75
+ # (see #debug)
76
+ # @param (see #debug)
77
+ # @!method(info(msg, *args))
78
+ def info(msg, *args)
79
+ self.message :INFO, msg, *args
80
+ end
81
+
82
+ # Send a warning message to the logger.
83
+ #
84
+ # (see #debug)
85
+ # @param (see #debug)
86
+ # @!method(warn(msg, *args))
87
+ def warn(msg, *args)
88
+ self.message :WARN, msg, *args
89
+ end
90
+
91
+ # Send an error message to the logger.
92
+ #
93
+ # (see #debug)
94
+ # @param (see #debug)
95
+ # @!method(error(msg, *args))
96
+ def error(msg, *args)
97
+ self.message :ERROR, msg, *args
98
+ end
99
+
100
+ # Send a fatal message to the logger.
101
+ #
102
+ # (see #debug)
103
+ # @param (see #debug)
104
+ # @!method(fatal_error(msg, *args))
105
+ def fatal_error(msg, *args)
106
+ self.message :FATAL, msg, *args
107
+ end
108
+
109
+ # The method that performs the code logging action.
110
+ #
111
+ # If extra arguments are supplied, the message string is expected to be a format specification string and the
112
+ # extra arguments will be applied to it.
113
+ #
114
+ # This default message method implementation uses the logger of ::Libis::Tools::Config. If an 'appname'
115
+ # parameter is defined in the Config object, it will be used as program name by the logger, otherwise the
116
+ # class name is taken.
117
+ #
118
+ # @param [{::Logger::Severity}] severity
119
+ # @param [String] msg message string
120
+ # @param [Object] args optional list of extra arguments
121
+ # @!method(message(severity, msg, *args))
122
+ def message(severity, msg, *args)
123
+ message_text = (msg % args rescue "#{msg}#{args.empty? ? '' : " - #{args}"}")
124
+ self.logger.add(::Logging.level_num(severity), message_text)
125
+ end
126
+
127
+ end
128
+
129
+ end
130
+ end