libis-tools 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/.coveralls.yml +2 -0
  3. data/.gitignore +16 -0
  4. data/.rspec +2 -0
  5. data/.travis.yml +37 -0
  6. data/Gemfile +7 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +289 -0
  9. data/Rakefile +6 -0
  10. data/lib/libis-tools.rb +1 -0
  11. data/lib/libis/tools.rb +16 -0
  12. data/lib/libis/tools/assert.rb +41 -0
  13. data/lib/libis/tools/checksum.rb +84 -0
  14. data/lib/libis/tools/command.rb +40 -0
  15. data/lib/libis/tools/config.rb +160 -0
  16. data/lib/libis/tools/dc_record.rb +47 -0
  17. data/lib/libis/tools/extend/empty.rb +7 -0
  18. data/lib/libis/tools/extend/hash.rb +107 -0
  19. data/lib/libis/tools/extend/ostruct.rb +3 -0
  20. data/lib/libis/tools/extend/string.rb +85 -0
  21. data/lib/libis/tools/extend/struct.rb +29 -0
  22. data/lib/libis/tools/logger.rb +71 -0
  23. data/lib/libis/tools/mets_file.rb +575 -0
  24. data/lib/libis/tools/parameter.rb +172 -0
  25. data/lib/libis/tools/sharepoint_mapping.rb +118 -0
  26. data/lib/libis/tools/sharepoint_record.rb +260 -0
  27. data/lib/libis/tools/version.rb +5 -0
  28. data/lib/libis/tools/xml_document.rb +574 -0
  29. data/libis-tools.gemspec +39 -0
  30. data/spec/assert_spec.rb +65 -0
  31. data/spec/checksum_spec.rb +132 -0
  32. data/spec/command_spec.rb +68 -0
  33. data/spec/config_spec.rb +86 -0
  34. data/spec/data/test.data +9 -0
  35. data/spec/data/test.xml +8 -0
  36. data/spec/data/test.yml +1 -0
  37. data/spec/logger_spec.rb +107 -0
  38. data/spec/parameter_container_spec.rb +83 -0
  39. data/spec/parameter_spec.rb +139 -0
  40. data/spec/spec_helper.rb +12 -0
  41. data/spec/test.xsd +20 -0
  42. data/spec/xmldocument_spec.rb +413 -0
  43. data/test/test_helper.rb +7 -0
  44. data/test/webservices/test_ca_item_info.rb +59 -0
  45. data/test/webservices/test_ca_search.rb +35 -0
  46. metadata +244 -0
@@ -0,0 +1,172 @@
1
+ # encoding: utf-8
2
+ require 'date'
3
+ require 'libis/tools/extend/struct'
4
+
5
+ module Libis
6
+ module Tools
7
+
8
+ # noinspection RubyConstantNamingConvention
9
+ Parameter = ::Struct.new(:name, :default, :datatype, :description, :constraint, :options) do
10
+
11
+ VALID_PARAMETER_KEYS = [:name, :default, :datatype, :description, :constraint, :options]
12
+
13
+ def initialize(*args)
14
+ # noinspection RubySuperCallWithoutSuperclassInspection
15
+ super(*args)
16
+ self.options = {} unless self.options
17
+ end
18
+
19
+ TRUE_BOOL = %w'true yes t y 1'
20
+ FALSE_BOOL = %w'false no f n 0'
21
+
22
+ def parse(value = nil)
23
+ result = if value.nil?
24
+ send(:default)
25
+ else
26
+ dtype = guess_datatype.to_s.downcase
27
+ convert(dtype, value)
28
+ end
29
+ check_constraint(result)
30
+ result
31
+ end
32
+
33
+ def valid_value?(value)
34
+ begin
35
+ parse(value)
36
+ rescue
37
+ return false
38
+ end
39
+ true
40
+ end
41
+
42
+ def guess_datatype
43
+ return send(:datatype) if send(:datatype)
44
+ case send(:default)
45
+ when TrueClass, FalseClass
46
+ 'bool'
47
+ when NilClass
48
+ 'string'
49
+ when Integer
50
+ 'int'
51
+ when Float
52
+ 'float'
53
+ when DateTime, Date, Time
54
+ 'datetime'
55
+ else
56
+ send(:default).class.name.downcase
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ def convert(dtype, v)
63
+ case dtype.to_s.downcase
64
+ when 'boolean', 'bool'
65
+ return true if TRUE_BOOL.include?(v.to_s.downcase)
66
+ return false if FALSE_BOOL.include?(v.to_s.downcase)
67
+ raise ArgumentError, "No boolean information in '#{v.to_s}'. Valid values are: '#{TRUE_BOOL.join('\', \'')}' and '#{FALSE_BOOL.join('\', \'')}'."
68
+ when 'string'
69
+ return v.to_s.gsub('%s', Time.now.strftime('%Y%m%d%H%M%S'))
70
+ when 'int'
71
+ return Integer(v)
72
+ when 'float'
73
+ return Float(v)
74
+ when 'datetime'
75
+ return v.to_datetime if v.respond_to? :to_datetime
76
+ return DateTime.parse(v)
77
+ else
78
+ raise RuntimeError, "Datatype not supported: '#{dtype}'"
79
+ end
80
+ end
81
+
82
+ def check_constraint(v, constraint = nil)
83
+ constraint ||= send(:constraint)
84
+ return if constraint.nil?
85
+ raise ArgumentError, "Value '#{v}' is not allowed (constraint: #{constraint})." unless constraint_checker(v, constraint)
86
+ end
87
+
88
+ def constraint_checker(v, constraint)
89
+
90
+ case constraint
91
+ when Array
92
+ constraint.each do |c|
93
+ return true if (constraint_checker(v, c) rescue false)
94
+ end
95
+ return true if constraint.include? v
96
+ when Range
97
+ return true if constraint.cover? v
98
+ when Regexp
99
+ return true if v =~ constraint
100
+ else
101
+ return true if v == constraint
102
+ end
103
+ false
104
+ end
105
+
106
+ end # Parameter
107
+
108
+ module ParameterContainer
109
+
110
+ module ClassMethods
111
+
112
+ def parameter(options = {})
113
+ if options.is_a? Hash
114
+ return nil if options.keys.empty?
115
+ param_def = options.shift
116
+ name = param_def.first.to_s.to_sym
117
+ default = param_def.last
118
+ parameters[name] = Parameter.new(name, default) if parameters[name].nil?
119
+ VALID_PARAMETER_KEYS.each { |key| parameters[name][key] = options[key] if options[key] }
120
+ else
121
+ parameters[options]
122
+ end
123
+ end
124
+
125
+ protected
126
+
127
+ def parameters
128
+ @parameters ||= Hash.new
129
+ end
130
+
131
+ end
132
+
133
+ def self.included(base)
134
+ base.extend(ClassMethods)
135
+ end
136
+
137
+ NO_VALUE = '##NAV##'
138
+
139
+ def parameter(name, value = NO_VALUE)
140
+ param_def = get_parameter_definition(name)
141
+ return NO_VALUE unless param_def
142
+ if value.equal? NO_VALUE
143
+ param_value = parameters[name]
144
+ param_def.parse(param_value)
145
+ else
146
+ return NO_VALUE unless param_def.valid_value?(value)
147
+ parameters[name] = value
148
+ end
149
+ end
150
+
151
+ def [](name)
152
+ parameter(name)
153
+ end
154
+
155
+ def []=(name, value)
156
+ parameter name, value
157
+ end
158
+
159
+ protected
160
+
161
+ def parameters
162
+ @parameters ||= Hash.new
163
+ end
164
+
165
+ def get_parameter_definition(name)
166
+ self.class.parameter(name)
167
+ end
168
+
169
+ end # ParameterContainer
170
+
171
+ end # Tools
172
+ end # Libis
@@ -0,0 +1,118 @@
1
+ # coding: utf-8
2
+
3
+ require 'csv'
4
+ require 'yaml'
5
+
6
+ require 'libis/tools/extend/hash'
7
+
8
+ module Libis
9
+ module Tools
10
+
11
+ class SharepointMapping < Hash
12
+
13
+ def initialize(mapping_file)
14
+
15
+
16
+ CSV.foreach(mapping_file, headers: true, skip_blanks: true) do |row|
17
+ next unless row[1]
18
+ # next unless (row[2] || row[3])
19
+
20
+ # compensation for bug in library that reads the Excel data
21
+ 0.upto(5) { |i| row[i] = row[i].gsub(/_x005F(_x[0-9a-fA-F]{4}_)/, '\1') if row[i] }
22
+
23
+ name = row[0] ? row[0].strip : nil
24
+ label = row[1].strip.to_sym
25
+ dc_tag = row[2] ? row[2].strip : ''
26
+ db_column = row[3] ? row[3].strip : nil
27
+ db_datatype = row[4] ? row[4].strip.upcase.to_sym : nil
28
+ db_valuemask = row[5] ? row[5] : nil
29
+ # scope_tag = row[6] ? row[6].strip : nil
30
+ # scope_id = (row[7] and row[7] =~ /[0-9]+/ ? Integer(row[7].strip) : nil)
31
+
32
+
33
+ mapping = {}
34
+ mapping[:fancy_name] = name if name
35
+ mapping[:db_column] = db_column if db_column
36
+ mapping[:db_datatype] = :STRING
37
+ mapping[:db_datatype] = db_datatype if db_datatype
38
+ mapping[:db_valuemask] = (mapping[:db_datatype] == :STRING ? "'@@'" : '@@')
39
+ mapping[:db_valuemask] = db_valuemask if db_valuemask
40
+ # mapping[:scope_tag] = scope_tag if scope_tag
41
+ # mapping[:scope_id] = scope_id if scope_id
42
+
43
+ if dc_tag.match(/^\s*"(.*)"\s*(<.*)$/)
44
+ mapping[:dc_prefix] = $1
45
+ dc_tag = $2
46
+ end
47
+
48
+ if dc_tag.match(/^\s*<dc:[^.]+\.([^.>]+)>(.*)$/)
49
+ mapping[:dc_tag] = "dcterms:#{$1}"
50
+ dc_tag = $2
51
+
52
+ elsif dc_tag.match(/^\s*<dc:([^.>]+)>(.*)$/)
53
+ mapping[:dc_tag] = "dc:#{$1}"
54
+ dc_tag = $2
55
+ end
56
+
57
+ if dc_tag.match(/^\s*"(.*)"\s*$/)
58
+ mapping[:dc_postfix] = $1
59
+ end
60
+
61
+ self[label] = mapping.empty? ? nil : mapping
62
+
63
+ end
64
+
65
+ File.open('mapping.yml', 'wt') { |fp|
66
+ fp.puts self.to_yaml
67
+ }
68
+ super nil
69
+
70
+ end
71
+
72
+ def name(label)
73
+ mapping = self[label]
74
+ mapping = mapping[:fancy_name] if mapping
75
+ mapping || label
76
+ end
77
+
78
+ def fancy_label(label)
79
+ mapping = self[label]
80
+ mapping = mapping[:fancy_name] if mapping
81
+ "#{label}#{mapping ? '(' + mapping + ')' : ''}"
82
+ end
83
+
84
+ def dc_tag(label)
85
+ mapping = self[label]
86
+ mapping = mapping[:dc_tag] if mapping
87
+ mapping
88
+ end
89
+
90
+ def dc_prefix(label)
91
+ mapping = self[label]
92
+ mapping = mapping[:dc_prefix] if mapping
93
+ mapping
94
+ end
95
+
96
+ def dc_postfix(label)
97
+ mapping = self[label]
98
+ mapping = mapping[:dc_postfix] if mapping
99
+ mapping
100
+ end
101
+
102
+ def db_column(label)
103
+ mapping = self[label]
104
+ mapping = mapping[:db_column] if mapping
105
+ mapping
106
+ end
107
+
108
+ def db_value(label, value)
109
+ mapping = self[label]
110
+ return nil unless mapping
111
+ mask = mapping[:db_valuemask]
112
+ mask.gsub('@@', value.to_s)
113
+ end
114
+
115
+ end
116
+
117
+ end
118
+ end
@@ -0,0 +1,260 @@
1
+ # coding: utf-8
2
+
3
+ require 'uri'
4
+
5
+ require 'libis/tools/extend/hash'
6
+ require 'libis/tools/xml_document'
7
+
8
+ module Libis
9
+ module Tools
10
+
11
+ # noinspection RubyTooManyMethodsInspection
12
+ class SharepointRecord < Hash
13
+
14
+ attr_accessor :node
15
+
16
+ def initialize
17
+ @node = nil
18
+ self[:label_prefix] = ''
19
+ super nil
20
+ end
21
+
22
+ def label_prefix
23
+ self[:label_prefix]
24
+ end
25
+
26
+ def label_prefix=(value)
27
+ self[:label_prefix] = value
28
+ end
29
+
30
+ def label
31
+ (self[:ows_Title1] || self[:ows_BaseName] || file_name).to_s
32
+ end
33
+
34
+ def title
35
+ self[:label_prefix] + ' ' + self.label
36
+ end
37
+
38
+ def content_type
39
+ self[:ows_ContentType]
40
+ end
41
+
42
+ def file_name
43
+ self[:ows_FileLeafRef]
44
+ end
45
+
46
+ def file_path
47
+ self[:ows_FileRef]
48
+ end
49
+
50
+ def file_size
51
+ self[:ows_FileSizeDisplay]
52
+ end
53
+
54
+ def url
55
+ # self[:ows_EncodedAbsUrl]
56
+ # 'https://www.groupware.kuleuven.be' + URI.escape(self[:ows_ServerUrl], Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
57
+ # 'https://www.groupware.kuleuven.be' + URI.escape(self[:ows_ServerUrl])
58
+ URI.escape(URI.unescape(self[:ows_EncodedAbsUrl]))
59
+ end
60
+
61
+ def relative_path
62
+ return file_path.gsub(/^sites\/lias\/Gedeelde documenten\//, '') if file_path
63
+ nil
64
+ end
65
+
66
+ def local_path(sub_dir)
67
+ return relative_path unless sub_dir
68
+ sub_dir += '/' unless sub_dir[-1] == '/'
69
+ return relative_path.gsub(/^#{sub_dir}/, '') if relative_path
70
+ nil
71
+ end
72
+
73
+ def is_file?
74
+ return true if [:file, :mfile].include? simple_content_type
75
+ false
76
+ end
77
+
78
+ def is_described?
79
+ =begin
80
+ self[:ows_Title1] and
81
+ ( self[:ows_Creation_x0020_date_x0028_s_x0029_] or
82
+ self[:ows_Startdate] or
83
+ self[:ows_Enddate]
84
+ )
85
+ =end
86
+ self[:ows_Unit_of_description]
87
+ end
88
+
89
+ def simple_content_type
90
+ case content_type
91
+ when /^Archief/i
92
+ return :archive
93
+ when /^Bestanddeel \(folder\)/i
94
+ return :map
95
+ when /^Bestanddeel of stuk \(document\)/i
96
+ return :file
97
+ when /^Meervoudige beschrijving \(folder\)/i
98
+ return :mmap
99
+ when /^Meervoudige beschrijving \(document\)/i
100
+ return :mfile
101
+ when /^Tussenniveau/i
102
+ return :map
103
+ when /^Film/i
104
+ return :file
105
+ when /^Object/i
106
+ return :file
107
+ when /^Document/i
108
+ return :file
109
+ else
110
+ # type code here
111
+ end
112
+ :unknown
113
+ end
114
+
115
+ def content_code
116
+ case simple_content_type
117
+ when :archive
118
+ 'a'
119
+ when :map
120
+ 'm'
121
+ when :file
122
+ 'f'
123
+ when :mmap
124
+ 'v'
125
+ when :mfile
126
+ '<'
127
+ when :unknown
128
+ '-'
129
+ else
130
+ ' '
131
+ end + (is_described? ? '*' : ' ')
132
+ end
133
+
134
+ def ingest_model
135
+ return self[:ows_Ingestmodel] if self[:ows_Ingestmodel]
136
+ return self.node.parent.content.ingest_model if node and node.parent and node.parent.content
137
+ nil
138
+ end
139
+
140
+ def accessright_model
141
+ return self[:ows_Access_x0020_rights_x0020_model] if self[:ows_Access_x0020_rights_x0020_model]
142
+ return self.node.parent.content.accessright_model if node and node.parent and node.parent.content
143
+ nil
144
+ end
145
+
146
+ def to_raw
147
+ self
148
+ end
149
+
150
+ def to_xml
151
+
152
+ xml_doc = Libis::Tools::XmlDocument.new
153
+
154
+ xml_doc.root = xml_doc.create_node('record')
155
+
156
+ self.each do |label, value|
157
+
158
+ unless label == :node
159
+ #noinspection RubyResolve
160
+ xml_doc.root << xml_doc.create_text_node(label.to_s, value.to_s)
161
+ end
162
+
163
+ end
164
+
165
+ xml_doc
166
+
167
+ end
168
+
169
+ def self.from_xml(xml_node)
170
+
171
+ record = Libis::Tools::SharepointRecord.new
172
+
173
+ xml_node.element_children.each do |node|
174
+ record[node.name.to_sym] = node.content
175
+ end
176
+
177
+ record
178
+
179
+ end
180
+
181
+ # @param [Libis::Tools::SharepointMapping] mapping
182
+ def to_dc(mapping)
183
+
184
+ return nil unless mapping and mapping.is_a? Hash
185
+
186
+ xml_doc = Libis::Tools::XmlDocument.new
187
+
188
+ #noinspection RubyStringKeysInHashInspection
189
+ xml_doc.root = xml_doc.create_node(
190
+ 'record',
191
+ namespaces: {
192
+ 'dc' => 'http://purl.org/dc/elements/1.1/',
193
+ 'xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
194
+ 'dcterms' => 'http://purl.org/dc/terms/'})
195
+
196
+ self.each do |label, value|
197
+ dc_tag = mapping.dc_tag(label)
198
+ next unless dc_tag
199
+ dc_value = (mapping.dc_prefix(label) || '') + value.to_s + (mapping.dc_postfix(label) || '')
200
+ #noinspection RubyResolve
201
+ xml_doc.root << xml_doc.create_text_node(dc_tag, dc_value)
202
+ end
203
+
204
+ if xml_doc.xpath('//dc:title').size == 0
205
+ xml_doc.root << xml_doc.create_text_node('dc:title', self[:ows_BaseName])
206
+ end
207
+
208
+ xml_doc
209
+
210
+ end
211
+
212
+ # @param [Libis::Tools::SharepointMapping] mapping
213
+ def to_sql(mapping)
214
+ sql_fields = []
215
+ sql_values = []
216
+
217
+ self.each do |label, value|
218
+ db_column = mapping.db_column(label)
219
+ next unless db_column
220
+ db_value = mapping.db_value(label, value)
221
+ next unless db_value and db_value != "''"
222
+ sql_fields << db_column
223
+ sql_values << db_value.escape_for_sql
224
+ end
225
+
226
+ sql_fields.each_with_index { |element, index| (index % 10 == 0) && (sql_fields[index] = "\n " + element)
227
+ }
228
+ sql_values.each_with_index { |element, index| (index % 10 == 0) && (sql_values[index] = "\n " + element)
229
+ }
230
+
231
+ 'INSERT INTO @TABLE_NAME@ (' + sql_fields.join(',') + ")\n VALUES (" + sql_values.join(',') + ');'
232
+
233
+ end
234
+
235
+ def create_dc(dir, mapping)
236
+ xml_doc = to_dc mapping
237
+ dc_file = "#{dir}/dc_#{self[:index].to_s}.xml"
238
+ xml_doc.save dc_file
239
+ dc_file
240
+ end
241
+
242
+ def to_s
243
+ super
244
+ end
245
+
246
+ def print_metadata(f, mapping)
247
+ f.printf "%6d -------------------------------------------------------------------------\n", self[:index].to_i
248
+ self.each do |label, value|
249
+ next if label == :node
250
+ # next if label == :index
251
+ name = mapping.fancy_label(label)
252
+ f.printf " %40s : %s\n", name, value
253
+ end
254
+
255
+ end
256
+
257
+ end
258
+
259
+ end
260
+ end