bulk_ops 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/app/assets/images/bulk_ops/github_logo.png +0 -0
- data/app/assets/javascripts/bulk_ops.js +14 -0
- data/app/assets/javascripts/bulk_ops/selections.js +24 -0
- data/app/assets/javascripts/selections.js +38 -0
- data/app/assets/javascripts/work_search.js +64 -0
- data/app/assets/stylesheets/bulk_ops.scss +99 -0
- data/app/controllers/bulk_ops/application_controller.rb +13 -0
- data/app/controllers/bulk_ops/github_authorization_controller.rb +33 -0
- data/app/controllers/bulk_ops/operations_controller.rb +481 -0
- data/app/jobs/bulk_ops/application_job.rb +4 -0
- data/app/mailers/bulk_ops/application_mailer.rb +6 -0
- data/app/models/bulk_ops/application_record.rb +5 -0
- data/app/views/bulk_ops/_bulk_ops_sidebar_widget.html.erb +15 -0
- data/app/views/bulk_ops/_github_auth_widget.html.erb +13 -0
- data/app/views/bulk_ops/operations/_bulk_ops_header.html.erb +4 -0
- data/app/views/bulk_ops/operations/_choose_fields.html.erb +22 -0
- data/app/views/bulk_ops/operations/_choose_notifications.html.erb +22 -0
- data/app/views/bulk_ops/operations/_git_message.html.erb +7 -0
- data/app/views/bulk_ops/operations/_ingest_options.html.erb +42 -0
- data/app/views/bulk_ops/operations/_operation_options.html.erb +38 -0
- data/app/views/bulk_ops/operations/_show_authorize.html.erb +13 -0
- data/app/views/bulk_ops/operations/_show_complete.html.erb +31 -0
- data/app/views/bulk_ops/operations/_show_draft.html.erb +20 -0
- data/app/views/bulk_ops/operations/_show_new.html.erb +2 -0
- data/app/views/bulk_ops/operations/_show_pending.html.erb +58 -0
- data/app/views/bulk_ops/operations/_show_running.html.erb +56 -0
- data/app/views/bulk_ops/operations/_show_verifying.html.erb +8 -0
- data/app/views/bulk_ops/operations/_show_waiting.html.erb +9 -0
- data/app/views/bulk_ops/operations/_update_draft_work_list.html.erb +45 -0
- data/app/views/bulk_ops/operations/_update_draft_work_search.html.erb +59 -0
- data/app/views/bulk_ops/operations/_update_options.html.erb +9 -0
- data/app/views/bulk_ops/operations/index.html.erb +51 -0
- data/app/views/bulk_ops/operations/new.html.erb +36 -0
- data/app/views/bulk_ops/operations/show.html.erb +7 -0
- data/config/routes.rb +25 -0
- data/db/migrate/20180926190757_create_github_credentials.rb +13 -0
- data/db/migrate/20181017180436_create_bulk_ops_tables.rb +40 -0
- data/lib/bulk_ops.rb +15 -0
- data/lib/bulk_ops/create_spreadsheet_job.rb +43 -0
- data/lib/bulk_ops/create_work_job.rb +14 -0
- data/lib/bulk_ops/delete_file_set_job.rb +15 -0
- data/lib/bulk_ops/engine.rb +6 -0
- data/lib/bulk_ops/error.rb +141 -0
- data/lib/bulk_ops/github_access.rb +284 -0
- data/lib/bulk_ops/github_credential.rb +3 -0
- data/lib/bulk_ops/operation.rb +358 -0
- data/lib/bulk_ops/relationship.rb +79 -0
- data/lib/bulk_ops/search_builder_behavior.rb +80 -0
- data/lib/bulk_ops/templates/configuration.yml +5 -0
- data/lib/bulk_ops/templates/readme.md +1 -0
- data/lib/bulk_ops/update_work_job.rb +14 -0
- data/lib/bulk_ops/verification.rb +210 -0
- data/lib/bulk_ops/verification_job.rb +23 -0
- data/lib/bulk_ops/version.rb +3 -0
- data/lib/bulk_ops/work_job.rb +104 -0
- data/lib/bulk_ops/work_proxy.rb +466 -0
- data/lib/generators/bulk_ops/install/install_generator.rb +27 -0
- data/lib/generators/bulk_ops/install/templates/config/github.yml.example +28 -0
- metadata +145 -0
@@ -0,0 +1,284 @@
|
|
1
|
+
require "octokit"
|
2
|
+
require "socket"
|
3
|
+
require "securerandom"
|
4
|
+
require 'base64'
|
5
|
+
|
6
|
+
class BulkOps::GithubAccess
|
7
|
+
|
8
|
+
ROW_OFFSET = 2
|
9
|
+
SPREADSHEET_FILENAME = 'metadata.csv'
|
10
|
+
OPTIONS_FILENAME = 'configuration.yml'
|
11
|
+
|
12
|
+
attr_accessor :name
|
13
|
+
|
14
|
+
def self.auth_url user
|
15
|
+
"#{Octokit.web_endpoint}login/oauth/authorize?client_id=#{client_id}&redirect_uri=#{redirect_endpoint(user)}&state=#{state(user)}&scope=repo"
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.username user
|
19
|
+
return false unless (cred = BulkOps::GithubCredential.find_by(user_id: user))
|
20
|
+
return false unless (token = cred.oauth_token)
|
21
|
+
Octokit::Client.new(access_token: token).user[:login] rescue false
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.valid_state?(astate,user_id=nil)
|
25
|
+
return (astate === state(user_id))
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.state user_id=nil
|
29
|
+
user_id = current_user.id if user_id.nil?
|
30
|
+
user_id = user_id.id if user_id.is_a? User
|
31
|
+
user_id = user_id.to_i if user_id.is_a? String
|
32
|
+
cred = BulkOps::GithubCredential.find_by(user_id: user_id) || BulkOps::GithubCredential.create({user_id: user_id, state: SecureRandom.hex })
|
33
|
+
cred.state
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.auth_token user_id=nil
|
37
|
+
user_id = current_user.id if user_id.nil?
|
38
|
+
user_id = user_id.id if user_id.is_a? User
|
39
|
+
user_id = user_id.to_i if user_id.is_a? String
|
40
|
+
return false unless (cred = BulkOps::GithubCredential.find_by(user_id: user_id))
|
41
|
+
return cred.auth_token || false
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.set_auth_token!(code,user_id=nil)
|
45
|
+
user_id = current_user.id if user_id.nil?
|
46
|
+
user_id = user_id.id if user_id.is_a? User
|
47
|
+
user_id = user_id.to_i if user_id.is_a? String
|
48
|
+
cred = BulkOps::GithubCredential.find_by(user_id: user_id)
|
49
|
+
result = HTTParty.post("https://github.com/login/oauth/access_token",
|
50
|
+
body: {client_id: client_id,
|
51
|
+
client_secret: client_secret,
|
52
|
+
code: code,
|
53
|
+
accept: "json"}.to_json,
|
54
|
+
headers: { 'Content-Type' => 'application/json',
|
55
|
+
'Accept' => 'application/json'})
|
56
|
+
cred.update(oauth_token: result.parsed_response["access_token"])
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.client_id
|
60
|
+
YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_id"]
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.client_secret
|
64
|
+
YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_secret"]
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.webhook_token
|
68
|
+
YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_secret"]
|
69
|
+
end
|
70
|
+
|
71
|
+
def self.redirect_endpoint user, ssl: false
|
72
|
+
host = Socket.gethostname
|
73
|
+
host = "localhost" if Rails.env.development? or Rails.env.test?
|
74
|
+
protocol = Rails.env.production? ? "https" : "http"
|
75
|
+
protocol = "https" if ssl
|
76
|
+
"#{protocol}://#{host}/bulk_ops/authorize/#{User.first.id}"
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.create_branch! name
|
80
|
+
self.new(name).create_branch!
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.add_file name, file_path, file_name = nil, message: false
|
84
|
+
self.new(name).add_file file_path, file_name, message: message
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.add_contents name, file_name, contents, message: false
|
88
|
+
self.new(name).add_contents file_name, contents, message
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.load_options name, branch: nil
|
92
|
+
self.new(name).load_options branch: branch
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.load_metadata branch:, return_headers: false
|
96
|
+
self.new(branch).load_metadata return_headers: return_headers
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.update_options name, options, message: false
|
100
|
+
self.new(name).update_options options, message: message
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.list_branches
|
104
|
+
self.new.list_branches
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.list_branch_names user
|
108
|
+
self.new.list_branch_names
|
109
|
+
end
|
110
|
+
|
111
|
+
def initialize(newname="dummy", user = nil)
|
112
|
+
@name = newname.parameterize
|
113
|
+
@user = user
|
114
|
+
end
|
115
|
+
|
116
|
+
def create_branch!
|
117
|
+
client.create_ref repo, "heads/#{name}", current_master_commit_sha
|
118
|
+
end
|
119
|
+
|
120
|
+
def delete_branch!
|
121
|
+
return false unless list_branch_names.include? name
|
122
|
+
client.delete_branch repo, name
|
123
|
+
end
|
124
|
+
|
125
|
+
def add_file file_path, file_name = nil, message: nil
|
126
|
+
file_name ||= File.basename(file_path)
|
127
|
+
# unless (file_name.downcase == "readme.md") || (file_name.downcase.include? "#{name}/")
|
128
|
+
file_name = File.join(name, file_name) unless file_name.downcase.include? "#{name.downcase}/"
|
129
|
+
message ||= "adding file #{file_name} to github branch #{name}"
|
130
|
+
client.create_contents(repo, file_name, message, file: file_path, branch: name)
|
131
|
+
end
|
132
|
+
|
133
|
+
def add_contents file_name, contents, message=false
|
134
|
+
message ||= "adding file #{file_name} to github branch #{name}"
|
135
|
+
begin
|
136
|
+
client.create_contents(repo, file_name, message, contents, branch: name)
|
137
|
+
rescue Octokit::UnprocessableEntity
|
138
|
+
sha = get_file_sha(file_name)
|
139
|
+
client.update_contents(repo, file_name, message, sha, contents, branch: name)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def add_new_spreadsheet file, message=false
|
144
|
+
if file.is_a? Tempfile
|
145
|
+
file.close
|
146
|
+
add_file file.path, SPREADSHEET_FILENAME, message: message
|
147
|
+
elsif file.is_a?(String) && File.file?(file)
|
148
|
+
add_file file, SPREADSHEET_FILENAME, message: message
|
149
|
+
elsif file.is_a? String
|
150
|
+
add_contents(spreadsheet_path, SPREADSHEET_FILENAME, message: message)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def list_branches
|
155
|
+
client.branches(repo).select{|branch| branch[:name] != "master"}
|
156
|
+
end
|
157
|
+
|
158
|
+
def list_branch_names
|
159
|
+
list_branches.map{|branch| branch[:name]}
|
160
|
+
end
|
161
|
+
|
162
|
+
def update_spreadsheet file, message: false
|
163
|
+
message ||= "updating metadata spreadsheet through hyrax browser interface."
|
164
|
+
sha = get_file_sha(spreadsheet_path)
|
165
|
+
file = File.new(file) if file.is_a?(String) && File.exist?(file)
|
166
|
+
client.update_contents(repo, spreadsheet_path, message, sha, file.read, branch: name)
|
167
|
+
end
|
168
|
+
|
169
|
+
def update_options options, message: false
|
170
|
+
if options['reference_column_name']
|
171
|
+
(options['ignored_columns'] ||= []) << options['reference_column_name']
|
172
|
+
(options['reference_identifier'] ||= []) << options['reference_column_name']
|
173
|
+
end
|
174
|
+
|
175
|
+
options['ignored_columns'] = options['ignored_columns'].reject { |c| c.empty? } if options['ignored_columns'].present?
|
176
|
+
|
177
|
+
message ||= "updating metadata spreadsheet through hyrax browser interface."
|
178
|
+
sha = get_file_sha(options_path)
|
179
|
+
client.update_contents(repo, options_path, message, sha, YAML.dump(options), branch: name)
|
180
|
+
end
|
181
|
+
|
182
|
+
def load_options branch: nil
|
183
|
+
branch ||= name
|
184
|
+
YAML.load(Base64.decode64(get_file_contents(options_path, ref: branch)))
|
185
|
+
end
|
186
|
+
|
187
|
+
def spreadsheet_count branch: nil
|
188
|
+
branch ||= name
|
189
|
+
Base64.decode64(get_file_contents(spreadsheet_path, ref: branch)).lines.count - 1
|
190
|
+
end
|
191
|
+
|
192
|
+
def load_metadata branch: nil, return_headers: false
|
193
|
+
branch ||= name
|
194
|
+
CSV.parse(Base64.decode64(get_file_contents(spreadsheet_path, ref: branch)), {headers: true, return_headers: return_headers})
|
195
|
+
end
|
196
|
+
|
197
|
+
def log_ingest_event log_level, row_number, event_type, message, commit_sha = nil
|
198
|
+
commit_sha ||= current_branch_commit_sha
|
199
|
+
#TODO WRITE THIS CODE
|
200
|
+
end
|
201
|
+
|
202
|
+
def create_pull_request message: false
|
203
|
+
begin
|
204
|
+
message ||= "Apply update #{name} through Hyrax browser interface"
|
205
|
+
pull = client.create_pull_request(repo, "master", name, message)
|
206
|
+
pull["number"]
|
207
|
+
rescue Octokit::UnprocessableEntity
|
208
|
+
return false
|
209
|
+
end
|
210
|
+
end
|
211
|
+
|
212
|
+
def can_merge?
|
213
|
+
return true
|
214
|
+
end
|
215
|
+
|
216
|
+
def merge_pull_request pull_id, message: false
|
217
|
+
client.merge_pull_request(repo, pull_id, message)
|
218
|
+
end
|
219
|
+
|
220
|
+
def get_metadata_row row_number
|
221
|
+
@current_metadata ||= load_metadata
|
222
|
+
@current_metadata[row_number - ROW_OFFSET]
|
223
|
+
end
|
224
|
+
|
225
|
+
def get_past_metadata_row commit_sha, row_number
|
226
|
+
past_metadata = Base64.decode64( client.contents(repo, path: filename, ref: commit_sha) )
|
227
|
+
past_metadata[row_number - ROW_OFFSET]
|
228
|
+
end
|
229
|
+
|
230
|
+
def get_file filename
|
231
|
+
client.contents(repo, path: filename, ref: name)
|
232
|
+
end
|
233
|
+
|
234
|
+
def get_file_contents filename, ref: nil
|
235
|
+
ref ||= name
|
236
|
+
client.contents(repo, path: filename, ref: ref)[:content]
|
237
|
+
end
|
238
|
+
|
239
|
+
def get_file_sha filename
|
240
|
+
client.contents(repo, path: filename, ref: name)[:sha]
|
241
|
+
end
|
242
|
+
|
243
|
+
def repo
|
244
|
+
github_config["repo"]
|
245
|
+
end
|
246
|
+
|
247
|
+
def spreadsheet_path
|
248
|
+
"#{name}/#{SPREADSHEET_FILENAME}"
|
249
|
+
end
|
250
|
+
|
251
|
+
private
|
252
|
+
|
253
|
+
def options_path
|
254
|
+
"#{name}/#{OPTIONS_FILENAME}"
|
255
|
+
end
|
256
|
+
|
257
|
+
def current_master_commit_sha
|
258
|
+
client.branch(repo,"master").commit.sha
|
259
|
+
end
|
260
|
+
|
261
|
+
def current_branch_commit_sha
|
262
|
+
client.branch(repo, name).commit.sha
|
263
|
+
end
|
264
|
+
|
265
|
+
def client
|
266
|
+
return @client if @client
|
267
|
+
return default_client if @user.nil?
|
268
|
+
return default_client unless (cred = BulkOps::GithubCredential.find_by(user_id: @user.id))
|
269
|
+
return default_client unless (token = cred.oauth_token)
|
270
|
+
client ||= Octokit::Client.new(access_token: token)
|
271
|
+
return default_client unless client.user[:login].is_a? String
|
272
|
+
@client = client
|
273
|
+
end
|
274
|
+
|
275
|
+
def default_client
|
276
|
+
@client ||= Octokit::Client.new(login: github_config["default_user"], password: github_config["default_password"])
|
277
|
+
end
|
278
|
+
|
279
|
+
def github_config
|
280
|
+
@github_config ||= YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]
|
281
|
+
end
|
282
|
+
|
283
|
+
|
284
|
+
end
|
@@ -0,0 +1,358 @@
|
|
1
|
+
require 'bulk_ops/verification'
|
2
|
+
module BulkOps
|
3
|
+
class Operation < ActiveRecord::Base
|
4
|
+
self.table_name = "bulk_ops_operations"
|
5
|
+
belongs_to :user
|
6
|
+
has_many :work_proxies, class_name: "::BulkOps::WorkProxy"
|
7
|
+
|
8
|
+
include BulkOps::Verification
|
9
|
+
|
10
|
+
attr_accessor :work_type, :visibility, :reference_identifier
|
11
|
+
|
12
|
+
delegate :can_merge?, :merge_pull_request, to: :git
|
13
|
+
|
14
|
+
INGEST_MEDIA_PATH = "/dams_ingest"
|
15
|
+
TEMPLATE_DIR = "lib/bulk_ops/templates"
|
16
|
+
RELATIONSHIP_COLUMNS = ["parent","child","next"]
|
17
|
+
SPECIAL_COLUMNS = ["parent",
|
18
|
+
"child",
|
19
|
+
"order",
|
20
|
+
"next",
|
21
|
+
"work_type",
|
22
|
+
"collection",
|
23
|
+
"collection_title",
|
24
|
+
"collection_id",
|
25
|
+
"visibility",
|
26
|
+
"relationship_identifier_type",
|
27
|
+
"id",
|
28
|
+
"filename",
|
29
|
+
"file"]
|
30
|
+
IGNORED_COLUMNS = ["ignore","offline_notes"]
|
31
|
+
OPTION_REQUIREMENTS = {type: {required: true,
|
32
|
+
values:[:ingest,:update]},
|
33
|
+
file_method: {required: :true,
|
34
|
+
values: [:replace_some,:add_remove,:replace_all]},
|
35
|
+
notifications: {required: true}}
|
36
|
+
|
37
|
+
def self.unique_name name, user
|
38
|
+
while BulkOps::Operation.find_by(name: name) || BulkOps::GithubAccess.list_branch_names(user).include?(name) do
|
39
|
+
if ['-','_'].include?(name[-2]) && name[-1].to_i > 0
|
40
|
+
name = name[0..-2]+(name[-1].to_i + 1).to_s
|
41
|
+
else
|
42
|
+
name = name + "_1"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
return name
|
46
|
+
end
|
47
|
+
|
48
|
+
def proxy_errors
|
49
|
+
work_proxies.reduce([]) do |errors, proxy|
|
50
|
+
if proxy.proxy_errors
|
51
|
+
errors += proxy.proxy_errors
|
52
|
+
elsif proxy.status == "job_error"
|
53
|
+
errors += BulkOps::Error.new(type: :job_failure, object_id: proxy.work_id, message: proxy.message)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def proxy_states
|
59
|
+
states = {}
|
60
|
+
work_proxies.each{|proxy| (states[proxy.status] ||= []) << proxy }
|
61
|
+
states
|
62
|
+
end
|
63
|
+
|
64
|
+
def type
|
65
|
+
operation_type
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.schema
|
69
|
+
ScoobySnacks::METADATA_SCHEMA
|
70
|
+
end
|
71
|
+
|
72
|
+
def schema
|
73
|
+
self.class.schema
|
74
|
+
end
|
75
|
+
|
76
|
+
def work_type
|
77
|
+
options["work_type"] || "work"
|
78
|
+
end
|
79
|
+
|
80
|
+
def reference_identifier
|
81
|
+
options["reference_identifier"] || "id/row"
|
82
|
+
end
|
83
|
+
|
84
|
+
def set_stage new_stage
|
85
|
+
update(stage: new_stage)
|
86
|
+
end
|
87
|
+
|
88
|
+
def apply!
|
89
|
+
status = "#{type}ing"
|
90
|
+
update({stage: "running", message: "#{type.titleize} initiated by #{user.name || user.email}"})
|
91
|
+
# @stage = "running"
|
92
|
+
final_spreadsheet
|
93
|
+
|
94
|
+
# This commented line currently fails because it doesn't pull from the master branch by default
|
95
|
+
# It's usually already verified, but maybe we should fix this for double-checking
|
96
|
+
# in the future
|
97
|
+
# return unless verify
|
98
|
+
|
99
|
+
apply_ingest! if ingest?
|
100
|
+
apply_update! if update?
|
101
|
+
end
|
102
|
+
|
103
|
+
def apply_ingest!
|
104
|
+
#Destroy any existing work proxies (which should not exist for an ingest). Create new proxies from finalized spreadsheet only.
|
105
|
+
work_proxies.each{|proxy| proxy.destroy!}
|
106
|
+
|
107
|
+
#create a work proxy for each row in the spreadsheet
|
108
|
+
@metadata.each_with_index do |values,row_number|
|
109
|
+
next if values.to_s.gsub(',','').blank?
|
110
|
+
work_proxies.create(status: "queued",
|
111
|
+
last_event: DateTime.now,
|
112
|
+
row_number: row_number,
|
113
|
+
message: "created during ingest initiated by #{user.name || user.email}")
|
114
|
+
end
|
115
|
+
# make sure the work proxies we just created are loaded in memory
|
116
|
+
reload
|
117
|
+
#loop through the work proxies to create a job for each work
|
118
|
+
@metadata.each_with_index do |values,row_number|
|
119
|
+
proxy = work_proxies.find_by(row_number: row_number)
|
120
|
+
proxy.update(message: "interpreted at #{DateTime.now.strftime("%d/%m/%Y %H:%M")} " + proxy.message)
|
121
|
+
data = proxy.interpret_data values
|
122
|
+
next unless proxy.proxy_errors.blank?
|
123
|
+
BulkOps::CreateWorkJob.perform_later(proxy.work_type || "Work",
|
124
|
+
user.email,
|
125
|
+
data,
|
126
|
+
proxy.id,
|
127
|
+
proxy.visibility || "open")
|
128
|
+
end
|
129
|
+
# If any errors have occurred, make sure they are logged in github and users are notified.
|
130
|
+
report_errors!
|
131
|
+
end
|
132
|
+
|
133
|
+
def delete_all
|
134
|
+
work_proxies.each do |proxy|
|
135
|
+
ActiveFedora::Base.find(proxy.work_id).destroy
|
136
|
+
proxy.update(status: "destroyed", message: "The work created by this proxy was destroyed by the user")
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def check_if_finished
|
141
|
+
return unless stage == "running" && !busy?
|
142
|
+
update(stage: accumulated_errors.blank? ? "complete" : "errors" )
|
143
|
+
report_errors!
|
144
|
+
lift_holds
|
145
|
+
end
|
146
|
+
|
147
|
+
def lift_holds
|
148
|
+
work_proxies.each { |proxy| proxy.lift_hold}
|
149
|
+
end
|
150
|
+
|
151
|
+
def place_holds
|
152
|
+
work_proxies.each { |proxy| proxy.place_hold}
|
153
|
+
end
|
154
|
+
|
155
|
+
def apply_update! spreadsheet
|
156
|
+
|
157
|
+
# this array will keep track of any current proxies not included in the final spreadsheet
|
158
|
+
abandoned_proxies = work_proxies.dup
|
159
|
+
# Loop through the final spreadsheet
|
160
|
+
final_spreadsheet.each_with_index do |values,row_number|
|
161
|
+
# Grab the work id
|
162
|
+
work_id = false
|
163
|
+
values.each{|field,val| work_id = val if ["id","workid","recordid"].include?(field.downcase.gsub(/-_\s/,''))}
|
164
|
+
@operation_errors << BulkOps::Error.new(:no_work_id_field) unless work_id
|
165
|
+
|
166
|
+
#proxy = BulkOps::WorkProxy.find_by(operation_id: id, work_id: values["work_id"])
|
167
|
+
if (proxy = work_proxies.find_by(work_id: work_id))
|
168
|
+
abandoned_proxies.delete(proxy)
|
169
|
+
proxy.update(status: "updating",
|
170
|
+
row_number: row_number,
|
171
|
+
message: "update initiated by #{user.name || user.email}")
|
172
|
+
else
|
173
|
+
# Create a proxy for a work that is in the spreadsheet, but wasn't in the initial draft
|
174
|
+
work_proxies.create(status: "queued",
|
175
|
+
last_event: DateTime.now,
|
176
|
+
row_number: row_number,
|
177
|
+
message: "created during update application, which was initiated by #{user.name || user.email}")
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
# Loop through any proxies in the draft that were dropped from the spreadsheet
|
182
|
+
abandoned_proxies.each do |dead_proxy|
|
183
|
+
dead_proxy.lift_hold
|
184
|
+
dead_proxy.destroy!
|
185
|
+
end
|
186
|
+
|
187
|
+
#loop through the work proxies to create a job for each work
|
188
|
+
work_proxies.each do |proxy|
|
189
|
+
data = proxy.interpret_data final_spreadsheet[proxy.row_number]
|
190
|
+
BulkOps::UpdateWorkJob.perform_later(proxy.work_type || "",
|
191
|
+
user.email,
|
192
|
+
data,
|
193
|
+
proxy.id,
|
194
|
+
proxy.visibility || "private")
|
195
|
+
end
|
196
|
+
report_errors!
|
197
|
+
end
|
198
|
+
|
199
|
+
def accumulated_errors
|
200
|
+
proxy_errors + (@operation_errors || [])
|
201
|
+
# TODO - make sure this captures all operation errors
|
202
|
+
end
|
203
|
+
|
204
|
+
def report_errors!
|
205
|
+
error_file_name = BulkOps::Error.write_errors!(accumulated_errors, git)
|
206
|
+
notify!(subject: "Errors initializing bulk #{type} in Hycruz", message: "Hycruz encountered some errors while it was setting up your #{type} and preparing to begin. For most types of errors, the individual rows of the spreadsheet with errors will be ignored and the rest will proceed. Please consult the #{type} summary for real time information on the status of the #{type}. Details about these initialization errors can be seen on Github at the following url: https://github.com/#{git.repo}/blob/#{git.name}/#{git.name}/errors/#{error_file_name}") if error_file_name
|
207
|
+
end
|
208
|
+
|
209
|
+
def create_pull_request message: false
|
210
|
+
return false unless (pull_num = git.create_pull_request(message: message))
|
211
|
+
update(pull_id: pull_num)
|
212
|
+
return pull_num
|
213
|
+
end
|
214
|
+
|
215
|
+
def finalize_draft(fields: nil, work_ids: nil)
|
216
|
+
create_new_spreadsheet(fields: fields, work_ids: work_ids)
|
217
|
+
update(stage: "pending")
|
218
|
+
end
|
219
|
+
|
220
|
+
def create_branch(fields: nil, work_ids: nil, options: nil, operation_type: :ingest)
|
221
|
+
git.create_branch!
|
222
|
+
bulk_ops_dir = Gem::Specification.find_by_name("bulk_ops").gem_dir
|
223
|
+
|
224
|
+
#copy template files
|
225
|
+
Dir["#{bulk_ops_dir}/#{TEMPLATE_DIR}/*"].each do |file|
|
226
|
+
git.add_file file
|
227
|
+
end
|
228
|
+
|
229
|
+
#update configuration options
|
230
|
+
unless options.blank?
|
231
|
+
full_options = YAML.load_file(File.join(bulk_ops_dir,TEMPLATE_DIR, BulkOps::GithubAccess::OPTIONS_FILENAME))
|
232
|
+
|
233
|
+
puts "Creating new bulk_ops options"
|
234
|
+
puts "new options:"
|
235
|
+
puts options.inspect
|
236
|
+
puts "standard options:"
|
237
|
+
puts full_options.inspect
|
238
|
+
|
239
|
+
options.each { |option, value| full_options[option] = value }
|
240
|
+
|
241
|
+
full_options[name] = name
|
242
|
+
full_options[type] = type
|
243
|
+
full_options[status] = status
|
244
|
+
|
245
|
+
puts "final options:"
|
246
|
+
puts full_options.inspect
|
247
|
+
|
248
|
+
git.update_options full_options
|
249
|
+
end
|
250
|
+
|
251
|
+
create_new_spreadsheet(fields: fields, work_ids: work_ids) if operation_type == :ingest
|
252
|
+
end
|
253
|
+
|
254
|
+
def get_spreadsheet return_headers: false
|
255
|
+
git.load_metadata return_headers: return_headers
|
256
|
+
end
|
257
|
+
|
258
|
+
def spreadsheet_count
|
259
|
+
git.spreadsheet_count
|
260
|
+
end
|
261
|
+
|
262
|
+
def final_spreadsheet
|
263
|
+
@metadata ||= git.load_metadata branch: "master"
|
264
|
+
end
|
265
|
+
|
266
|
+
def update_spreadsheet file, message: nil
|
267
|
+
git.update_spreadsheet(file, message: message)
|
268
|
+
end
|
269
|
+
|
270
|
+
def update_options filename, message=nil
|
271
|
+
git.update_options(filename, message: message)
|
272
|
+
end
|
273
|
+
|
274
|
+
def options
|
275
|
+
return {} if name.nil?
|
276
|
+
return @options if @options
|
277
|
+
branch = running? ? "master" : nil
|
278
|
+
@options ||= git.load_options(branch: branch)
|
279
|
+
end
|
280
|
+
|
281
|
+
def draft?
|
282
|
+
return (stage == 'draft')
|
283
|
+
end
|
284
|
+
|
285
|
+
def running?
|
286
|
+
return (stage == 'running')
|
287
|
+
end
|
288
|
+
|
289
|
+
def complete?
|
290
|
+
return (stage == 'complete')
|
291
|
+
end
|
292
|
+
|
293
|
+
def busy?
|
294
|
+
return true if work_proxies.where(status: "running").count > 0
|
295
|
+
return true if work_proxies.where(status: "queued").count > 0
|
296
|
+
return true if work_proxies.where(status: "starting").count > 0
|
297
|
+
return false
|
298
|
+
end
|
299
|
+
|
300
|
+
def ingest?
|
301
|
+
type == "ingest"
|
302
|
+
end
|
303
|
+
|
304
|
+
def update?
|
305
|
+
type == "update"
|
306
|
+
end
|
307
|
+
|
308
|
+
def delete_branch
|
309
|
+
git.delete_branch!
|
310
|
+
end
|
311
|
+
|
312
|
+
def destroy
|
313
|
+
git.delete_branch!
|
314
|
+
super
|
315
|
+
end
|
316
|
+
|
317
|
+
def self.default_metadata_fields(labels = true)
|
318
|
+
#returns full set of metadata parameters from ScoobySnacks to include in ingest template spreadsheet
|
319
|
+
field_names = []
|
320
|
+
schema.all_fields.each do |field|
|
321
|
+
field_names << field.name
|
322
|
+
field_names << "#{field.name} Label" if labels && field.controlled?
|
323
|
+
end
|
324
|
+
return field_names
|
325
|
+
end
|
326
|
+
|
327
|
+
def ignored_fields
|
328
|
+
(options['ignored headers'] || []) + IGNORED_COLUMNS
|
329
|
+
end
|
330
|
+
|
331
|
+
|
332
|
+
def error_url
|
333
|
+
"https://github.com/#{git.repo}/tree/#{git.name}/#{git.name}/errors"
|
334
|
+
end
|
335
|
+
|
336
|
+
def filename_prefix
|
337
|
+
@filename_prefix ||= options['filename_prefix']
|
338
|
+
end
|
339
|
+
|
340
|
+
private
|
341
|
+
|
342
|
+
def git
|
343
|
+
@git ||= BulkOps::GithubAccess.new(name, @user)
|
344
|
+
end
|
345
|
+
|
346
|
+
def create_new_spreadsheet(fields: nil, work_ids: nil)
|
347
|
+
work_ids ||= work_proxies.map{|proxy| proxy.work_id}
|
348
|
+
fields ||= self.class.default_metadata_fields
|
349
|
+
work_ids = [] if work_ids.nil?
|
350
|
+
if work_ids.count < 50
|
351
|
+
BulkOps::CreateSpreadsheetJob.perform_now(git.name, work_ids, fields, user)
|
352
|
+
else
|
353
|
+
BulkOps::CreateSpreadsheetJob.perform_later(git.name, work_ids, fields, user)
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
end
|
358
|
+
end
|