cdm_migrator 3.2.0 → 3.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/controllers/cdm_migrator/csv_controller.rb +40 -3
- data/app/views/cdm_migrator/cdm/mappings.html.erb +1 -1
- data/app/views/cdm_migrator/csv/_error_list.html.erb +21 -0
- data/lib/cdm_migrator/version.rb +1 -1
- metadata +7 -9
- data/app/views/cdm_migrator/csv/_path_list.html.erb +0 -19
- data/app/views/cdm_migrator/csv/_results_pagination.html.erb +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8bc6e448326179791de466ba5dc1d35a26fd48fa2a85a8fbabeb7d8dad60339e
|
4
|
+
data.tar.gz: f3cf46dec46cd8166bc0555eef28e2868db6fbbf77963083f93fca1958b00e19
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d8b8bc4d318269375c82429cbcd4bb2e3d07b1761ef54854b262746c09e46430ee3d9da02122567f5244599bb3f312b7248b1b0a610b12abc2a049b0b7470dc
|
7
|
+
data.tar.gz: 4dfc8a920f09e1e3c84b5eb09116adadcbf44556c3abe4b4daa1dbce7e87e767ea380de9fbdc5368c548032bf4b6f074a5ab0268fd6600dff75a9530445a7210
|
@@ -117,6 +117,7 @@ module CdmMigrator
|
|
117
117
|
solr = RSolr.connect url: Account.find_by(tenant: Apartment::Tenant.current).solr_endpoint.url
|
118
118
|
response = solr.get 'select', params: {
|
119
119
|
q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
|
120
|
+
rows: 3400,
|
120
121
|
fl: "id"
|
121
122
|
}
|
122
123
|
unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
|
@@ -269,13 +270,49 @@ module CdmMigrator
|
|
269
270
|
def check_edtf(row_number, row)
|
270
271
|
edtf_fields = @edtf_fields
|
271
272
|
edtf_errors = edtf_fields.each_with_object({}) do |field, hash|
|
272
|
-
|
273
|
-
|
273
|
+
temp_date = row[field]
|
274
|
+
# modify date so that the interval encompasses the years on the last interval date
|
275
|
+
temp_date = temp_date.gsub('/..','').gsub('%','?~').gsub(/\/$/,'')
|
276
|
+
date = temp_date.include?("/") ? temp_date.gsub(/([0-9]+X+\/)([0-9]+)(X+)/){"#{$1}"+"#{$2.to_i+1}"+"#{$3}"}.gsub("X","u") : temp_date
|
277
|
+
date = date.gsub("XX-","uu-").gsub("X-", "u-").gsub('XX?','uu').gsub('X?', 'u').gsub('u?','u').gsub('?','')
|
278
|
+
# edtf has trouble with year-month (e.g. "19uu-12") or year-season strings (e.g. "190u-23")
|
279
|
+
# that contain unspecified years, or intervals containing the above ("19uu-22/19uu-23", etc.).
|
280
|
+
# So we check for/create exceptions.
|
281
|
+
# Check for season interval
|
282
|
+
if Date.edtf(date) == nil and date != "unknown" # Accept season intervals
|
283
|
+
unless is_season?(date.split("/").first) and is_season?(date.split("/").second)
|
284
|
+
# If an interval then, check each date individually
|
285
|
+
if date.include?("/")
|
286
|
+
dates = date.split("/")
|
287
|
+
else
|
288
|
+
dates = [date]
|
289
|
+
end
|
290
|
+
#byebug
|
291
|
+
dates.each do |d|
|
292
|
+
# Dates with 'u' in the last digit of the year return invalid when in format YYYY-MM
|
293
|
+
# So we flub day specifity before checking again if the date is valid
|
294
|
+
unless Date.edtf(d + '-01') # Date.edtf('193u-03-01') returns valid
|
295
|
+
if match = d[/\d{3}u/] or match = d[/\d{2}u{2}-[2][1-4]/] # edtf can't parse single u in year (e.g. 192u) or uu in YYYY-SS (e.g. 19uu-21), so we replace it
|
296
|
+
d.gsub!(match, match.gsub("u","0"))
|
297
|
+
unless Date.edtf(d)
|
298
|
+
hash[field.to_s] = "Blank or not a valid EDTF date."
|
299
|
+
end
|
300
|
+
else
|
301
|
+
hash[field.to_s] = "Blank or not a valid EDTF date."
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
end
|
274
306
|
end
|
307
|
+
|
275
308
|
end
|
276
309
|
@error_list[row_number] = edtf_errors
|
277
310
|
end
|
278
311
|
|
312
|
+
def is_season?(date)
|
313
|
+
Date.edtf(date).class == EDTF::Season
|
314
|
+
end
|
315
|
+
|
279
316
|
# <Example: should be http://rightsstatements.org/vocab/etc. NOT https://rightsstatements.org/page/etc.
|
280
317
|
def check_uris(row_number, row)
|
281
318
|
uri_fields = @uri_fields
|
@@ -294,7 +331,7 @@ module CdmMigrator
|
|
294
331
|
value = row[field]
|
295
332
|
if value.present?
|
296
333
|
URI.extract(value).each { |uri| value.gsub!(uri, '') }
|
297
|
-
unless value.split("").all?
|
334
|
+
unless value.split("").all? { |sep| sep == character } # Check if remaining characters are the correct separator
|
298
335
|
hash[field.to_s] = "May contain the wrong multi-value separator (i.e. not #{character})."
|
299
336
|
end
|
300
337
|
end
|
@@ -30,7 +30,7 @@ border:1px solid black;
|
|
30
30
|
</tr>
|
31
31
|
<% end %>
|
32
32
|
</table>
|
33
|
-
<%= select_tag "mappings_url", options_for_select(@dirs) if @cdm_dirs %>
|
33
|
+
<%= select_tag "mappings_url", options_for_select(@dirs.sort { |x,y| x[0].downcase <=> y[0].downcase }) if @cdm_dirs %>
|
34
34
|
<%= hidden_field_tag "work", params[:work] %>
|
35
35
|
<%= submit_tag 'generate csv'%>
|
36
36
|
<% end %>
|
@@ -0,0 +1,21 @@
|
|
1
|
+
<% if @error_list && @error_list.any? %>
|
2
|
+
<%#= @error_list.inspect %>
|
3
|
+
<table class="table table-striped">
|
4
|
+
<thead>
|
5
|
+
<tr>
|
6
|
+
<th scope="col" style="min-width: 100px;">Line No.</th>
|
7
|
+
<th scope="col" style="margin-right: 0.5em;">Column(s)</th>
|
8
|
+
<th scope="col">Issue</th>
|
9
|
+
</tr>
|
10
|
+
</thead>
|
11
|
+
<tbody>
|
12
|
+
<% @error_list.keys.each do |line_number| %>
|
13
|
+
<tr>
|
14
|
+
<td><%= line_number %></td>
|
15
|
+
<td><%= @error_list[line_number].keys.join("<br />").html_safe %></td>
|
16
|
+
<td><%= @error_list[line_number].values.join("<br />").html_safe %></td>
|
17
|
+
</tr>
|
18
|
+
<% end %>
|
19
|
+
</tbody>
|
20
|
+
</table>
|
21
|
+
<% end %>
|
data/lib/cdm_migrator/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cdm_migrator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.2.
|
4
|
+
version: 3.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sephirothkod
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-11-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|
@@ -100,9 +100,8 @@ files:
|
|
100
100
|
- app/views/cdm_migrator/cdm/mappings.html.erb
|
101
101
|
- app/views/cdm_migrator/csv/_batches_list.html.erb
|
102
102
|
- app/views/cdm_migrator/csv/_default_group.html.erb
|
103
|
+
- app/views/cdm_migrator/csv/_error_list.html.erb
|
103
104
|
- app/views/cdm_migrator/csv/_list_batches.html.erb
|
104
|
-
- app/views/cdm_migrator/csv/_path_list.html.erb
|
105
|
-
- app/views/cdm_migrator/csv/_results_pagination.html.erb
|
106
105
|
- app/views/cdm_migrator/csv/_tabs.html.erb
|
107
106
|
- app/views/cdm_migrator/csv/csv_checker.html.erb
|
108
107
|
- app/views/cdm_migrator/csv/edit.html.erb
|
@@ -125,7 +124,7 @@ homepage: https://github.com/UVicLibrary/cdm_migrator
|
|
125
124
|
licenses:
|
126
125
|
- MIT
|
127
126
|
metadata: {}
|
128
|
-
post_install_message:
|
127
|
+
post_install_message:
|
129
128
|
rdoc_options: []
|
130
129
|
require_paths:
|
131
130
|
- lib
|
@@ -140,9 +139,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
139
|
- !ruby/object:Gem::Version
|
141
140
|
version: '0'
|
142
141
|
requirements: []
|
143
|
-
|
144
|
-
|
145
|
-
signing_key:
|
142
|
+
rubygems_version: 3.1.2
|
143
|
+
signing_key:
|
146
144
|
specification_version: 4
|
147
145
|
summary: ContentDM to Hyrax migrator.
|
148
146
|
test_files: []
|
@@ -1,19 +0,0 @@
|
|
1
|
-
<% if @path_list && @path_list.any? %>
|
2
|
-
<table class="table table-striped">
|
3
|
-
<thead>
|
4
|
-
<tr>
|
5
|
-
<th scope="col" style="min-width: 100px;">Line No.</th>
|
6
|
-
<th scope="col">File Path (url)</th>
|
7
|
-
</tr>
|
8
|
-
</thead>
|
9
|
-
<tbody>
|
10
|
-
<% @path_list.each do |line, path| %>
|
11
|
-
<tr>
|
12
|
-
<td><%= line %></td>
|
13
|
-
<td><%= path %></td>
|
14
|
-
</tr>
|
15
|
-
<% end %>
|
16
|
-
</tbody>
|
17
|
-
</table>
|
18
|
-
<% end %>
|
19
|
-
|