bulk_ops 0.1.15 → 0.1.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/bulk_ops/github_access.rb +10 -10
- data/lib/bulk_ops/operation.rb +5 -1
- data/lib/bulk_ops/parser.rb +41 -7
- data/lib/bulk_ops/relationship.rb +1 -1
- data/lib/bulk_ops/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2951c792e8835750322d76ea69f9c81f4e5dd37190bfbc235c3b14f5f166de5
|
4
|
+
data.tar.gz: d2c26310a619ddd0476bb796d401fd7e1bb0f4659301ec69d8875eba230e75df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 77e34cc192aedbd99549f7a1296019e164218e03e7c80356ccc30bcb62cb0756835b07b6ea43d5efb8edc71dcf8277e214c941e788277b4a32a4ffaf70c4bd3f
|
7
|
+
data.tar.gz: 8d4f3d182a8e298b51090c04fb470899d4c4075e207f75a61d1108eb22b78983e8d8baa48cf017b04e822c801f48131ef5efd08f068639284b6740e39ecead62
|
@@ -212,16 +212,16 @@ class BulkOps::GithubAccess
|
|
212
212
|
client.merge_pull_request(repo, pull_id, message)
|
213
213
|
end
|
214
214
|
|
215
|
-
def get_metadata_row row_number
|
216
|
-
@current_metadata ||= load_metadata
|
217
|
-
@current_metadata[row_number - BulkOps::ROW_OFFSET]
|
218
|
-
end
|
219
|
-
|
220
|
-
def get_past_metadata_row commit_sha, row_number
|
221
|
-
past_metadata = Base64.decode64( client.contents(repo, path: filename, ref: commit_sha) )
|
222
|
-
past_metadata[row_number - BulkOps::ROW_OFFSET]
|
223
|
-
end
|
224
|
-
|
215
|
+
# def get_metadata_row row_number
|
216
|
+
# @current_metadata ||= load_metadata
|
217
|
+
# @current_metadata[row_number - BulkOps::ROW_OFFSET]
|
218
|
+
# end
|
219
|
+
#
|
220
|
+
# def get_past_metadata_row commit_sha, row_number
|
221
|
+
# past_metadata = Base64.decode64( client.contents(repo, path: filename, ref: commit_sha) )
|
222
|
+
# past_metadata[row_number - BulkOps::ROW_OFFSET]
|
223
|
+
# end
|
224
|
+
#
|
225
225
|
def get_file filename
|
226
226
|
client.contents(repo, path: filename, ref: name)
|
227
227
|
end
|
data/lib/bulk_ops/operation.rb
CHANGED
@@ -81,15 +81,19 @@ module BulkOps
|
|
81
81
|
#Destroy any existing work proxies (which should not exist for an ingest). Create new proxies from finalized spreadsheet only.
|
82
82
|
work_proxies.each{|proxy| proxy.destroy!}
|
83
83
|
|
84
|
-
#create a work proxy for each
|
84
|
+
#create a work proxy for each work in the spreadsheet, creating filesets where appropriate
|
85
85
|
@metadata.each_with_index do |values,row_number|
|
86
86
|
next if values.to_s.gsub(',','').blank?
|
87
|
+
|
88
|
+
next if BulkOps::Parser.is_file_set? @metadata, row_number
|
89
|
+
|
87
90
|
work_proxies.create(status: "queued",
|
88
91
|
last_event: DateTime.now,
|
89
92
|
row_number: row_number,
|
90
93
|
visibility: options['visibility'],
|
91
94
|
message: "created during ingest initiated by #{user.name || user.email}")
|
92
95
|
end
|
96
|
+
|
93
97
|
# make sure the work proxies we just created are loaded in memory
|
94
98
|
reload
|
95
99
|
#loop through the work proxies to create a job for each work
|
data/lib/bulk_ops/parser.rb
CHANGED
@@ -5,6 +5,18 @@ class BulkOps::Parser
|
|
5
5
|
|
6
6
|
delegate :relationships, :operation, :row_number, :work_id, :visibility, :work_type, :reference_identifier, :order, to: :proxy
|
7
7
|
|
8
|
+
def self.is_file_set? metadata, row_number
|
9
|
+
return false unless metadata[row_number].present?
|
10
|
+
# If there are any valid fields other than relationship or file fields, it is a work
|
11
|
+
metadata[row_number].each do |field, value|
|
12
|
+
next if is_file_field?(field)
|
13
|
+
next if ["parent", "order"].include(normalize_relationship_field_name(field))
|
14
|
+
next if ["title","label"].include(field.downcase.strip)
|
15
|
+
return false
|
16
|
+
end
|
17
|
+
return true
|
18
|
+
end
|
19
|
+
|
8
20
|
def initialize prx, metadata_sheet=nil
|
9
21
|
@proxy = prx
|
10
22
|
@raw_data = (metadata_sheet || proxy.operation.metadata)
|
@@ -18,9 +30,11 @@ class BulkOps::Parser
|
|
18
30
|
@proxy = proxy if proxy.present?
|
19
31
|
@raw_data = raw_data if raw_data.present?
|
20
32
|
setAdminSet
|
33
|
+
#The order here matters a little: interpreting the relationship fields specifies containing collections,
|
34
|
+
# which may have opinions about whether we should inherit metadata from parent works
|
35
|
+
interpret_relationship_fields
|
21
36
|
setMetadataInheritance
|
22
37
|
interpret_option_fields
|
23
|
-
interpret_relationship_fields
|
24
38
|
disambiguate_columns
|
25
39
|
interpret_file_fields
|
26
40
|
interpret_controlled_fields
|
@@ -150,7 +164,6 @@ class BulkOps::Parser
|
|
150
164
|
# some or all existing files, those replacement-related deletions are handled
|
151
165
|
# by the BulkOps::Operation.
|
152
166
|
#
|
153
|
-
# TODO: THIS DOES NOT YET MANAGE THE ORDER OF INGESTED FILESETS
|
154
167
|
|
155
168
|
row = @raw_row.dup
|
156
169
|
@raw_row.each do |field, value|
|
@@ -159,7 +172,6 @@ class BulkOps::Parser
|
|
159
172
|
#If our CSV interpreter is feeding us the headers as a line, ignore it.
|
160
173
|
next if field == value
|
161
174
|
|
162
|
-
|
163
175
|
# Check if this is a file field, and whether we are removing or adding a file
|
164
176
|
next unless (action = is_file_field?(field))
|
165
177
|
|
@@ -184,6 +196,22 @@ class BulkOps::Parser
|
|
184
196
|
end
|
185
197
|
end
|
186
198
|
end
|
199
|
+
|
200
|
+
# Check if any of the upcoming rows are child filesets
|
201
|
+
i = 1
|
202
|
+
while self.class.is_file_set?(@metadata,row_number+i)
|
203
|
+
child_row.each do |field,value|
|
204
|
+
next if value.blank?
|
205
|
+
title = value if ["title","label"].include?(field.downcase.strip)
|
206
|
+
if is_file_field?(field)
|
207
|
+
operation.get_file_paths(value).each do |filepath|
|
208
|
+
uploaded_file = Hyrax::UploadedFile.create(file: File.open(filepath), user: operation.user)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
i+=1
|
213
|
+
end
|
214
|
+
|
187
215
|
end
|
188
216
|
@raw_row = row
|
189
217
|
end
|
@@ -259,7 +287,7 @@ class BulkOps::Parser
|
|
259
287
|
|
260
288
|
# correctly interpret the notation "id:a78C2d81"
|
261
289
|
identifier_type, object_identifier = interpret_relationship_value(identifier_type, value)
|
262
|
-
|
290
|
+
|
263
291
|
relationship_parameters = { work_proxy_id: @proxy.id,
|
264
292
|
identifier_type: identifier_type,
|
265
293
|
relationship_type: relationship_type,
|
@@ -302,14 +330,20 @@ class BulkOps::Parser
|
|
302
330
|
def interpret_relationship_value id_type, value, field="parent"
|
303
331
|
#Handle "id:20kj4259" syntax if it hasn't already been handled
|
304
332
|
if (split = value.to_s.split(":")).count == 2
|
305
|
-
id_type = split.first
|
333
|
+
id_type, value = split.first
|
306
334
|
value = split.last
|
307
335
|
end
|
308
336
|
#Handle special shorthand syntax for refering to relative row numbers
|
309
337
|
if id_type == "row"
|
310
|
-
if value
|
338
|
+
#if the value is an integer
|
339
|
+
if value =~ /\A[-+]?[0-9]+\z/
|
340
|
+
if value.to_i < 0
|
311
341
|
# if given a negative integer, count backwards from the current row (remember that value.to_i is negative)
|
312
|
-
|
342
|
+
return [id_type,row_number + value.to_i]
|
343
|
+
elsif value.to_i > 0
|
344
|
+
# if given a positive integer, remove the row offset
|
345
|
+
value = (value.to_i - BulkOps::ROW_OFFSET).to_s
|
346
|
+
end
|
313
347
|
elsif value.to_s.downcase.include?("prev")
|
314
348
|
# if given any variation of the word "previous", get the first preceding row with no parent of its own
|
315
349
|
return [id_type,find_previous_parent(field)]
|
@@ -41,7 +41,7 @@ class BulkOps::Relationship < ActiveRecord::Base
|
|
41
41
|
return ActiveFedora::Base.find(objects.first["id"])
|
42
42
|
when "row"
|
43
43
|
object_proxy = BulkOps::WorkProxy.find_by(operation_id: work_proxy.operation_id,
|
44
|
-
row_number: (object_identifier.to_i
|
44
|
+
row_number: (object_identifier.to_i))
|
45
45
|
ActiveFedora::Base.find(object_proxy.work_id)
|
46
46
|
when "proxy_id"
|
47
47
|
return false unless (proxy = BulkOps::WorkProxy.find(proxy_id))
|
data/lib/bulk_ops/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulk_ops
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ned Henry, UCSC Library Digital Initiatives
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-10-
|
11
|
+
date: 2019-10-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rails
|