cdm_migrator 3.0.3 → 3.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b736448ec514cc585e82de8b36fc2139786ae824ec728df2b42e3e7a16906621
4
- data.tar.gz: 33a754dfe77cc91dbaa5a88d76c8c2c781c753b593eb1b49495bebe0fbf2f6a3
3
+ metadata.gz: cdf5a7cb1cb95ce144dd931ed6b309ebc20d5f93ee8e91e6cfbabafded6a35fb
4
+ data.tar.gz: 6594d9da22ee42db51c96ac5393289b210c5b60e2e0a3066aa8f5e3571d6987e
5
5
  SHA512:
6
- metadata.gz: 799f4b3e23cd5fdd96dcad93f787c292f52f67466b07eed8f1fd7b609f50d4e59cfb0de599e6f2ed4503baf2d0c298e5ae49f4e03604075e3b15c5c09db5babd
7
- data.tar.gz: aaa14ed1e586d57a027213a2f6a93c69f1020f6167e2d0d51bd1825a0c66e4b94236804006c34071b88560f37d6f6685713c6c4befcc0a0c3d0336f2ec2cda77
6
+ metadata.gz: 2798dd9b01e019d298cf007604fbb54105856e1761eb7d5cca62b7bd1d1f4587b4b48b0d43e9881ffb0a01ebf8d4a27b7b3469d83a68c265353368ebca4b2489
7
+ data.tar.gz: 0ab6fa9e3a5324fb0c80692cdcda02aee0fb18dd0b0d20a570789e3478292987a91101cbd86e362d929c56d197af6ed20f82372d2b275ecd69f1f4a5dda3f076
@@ -1,20 +1,20 @@
1
1
  module CdmMigrator
2
- class CsvController < ApplicationController
3
- helper_method :default_page_title, :admin_host?, :available_translations, :available_works
2
+ class CsvController < ApplicationController
3
+ helper_method :default_page_title, :admin_host?, :available_translations, :available_works
4
4
  include ActionView::Helpers::UrlHelper
5
- layout 'hyrax/dashboard' if Hyrax
5
+ layout 'hyrax/dashboard' if Hyrax
6
6
  before_action :authenticate, except: :index
7
7
 
8
8
  def file_path_checker
9
- if params[:file]
10
- check_paths params[:file].path
11
-
12
- if @path_list.blank?
13
- flash[:notice] = "All file paths are valid."
14
- else
15
- flash[:error] = "Cdm Migrator couldn't find files at the following urls. Please correct the paths and try again."
16
- end
17
- end
9
+ if params[:file]
10
+ check_paths params[:file].path
11
+
12
+ if @path_list.blank?
13
+ flash[:notice] = "All file paths are valid."
14
+ else
15
+ flash[:error] = "Cdm Migrator couldn't find files at the following urls. Please correct the paths and try again."
16
+ end
17
+ end
18
18
  end
19
19
 
20
20
  def index
@@ -27,19 +27,19 @@ module CdmMigrator
27
27
  end
28
28
  end
29
29
 
30
- def upload
31
- @admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
32
- @collections = Collection.all.map { |col| [col.title.first, col.id] }
33
- end
30
+ def upload
31
+ @admin_sets = AdminSet.all.map { |as| [as.title.first, as.id] }
32
+ @collections = Collection.all.map { |col| [col.title.first, col.id] }
33
+ end
34
34
 
35
- def create
36
- dir = Rails.root.join('public', 'uploads', 'csvs')
37
- FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
38
- time = DateTime.now.strftime('%s')
39
- filename = params[:csv_import][:csv_file].original_filename.gsub('.csv',"#{time}.csv")
40
- csv = dir.join(filename).to_s
41
- File.open(csv, 'wb') do |file|
42
- file.write(params[:csv_import][:csv_file].read)
35
+ def create
36
+ dir = Rails.root.join('public', 'uploads', 'csvs')
37
+ FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
38
+ time = DateTime.now.strftime('%s')
39
+ filename = params[:csv_import][:csv_file].original_filename.gsub('.csv', "#{time}.csv")
40
+ csv = dir.join(filename).to_s
41
+ File.open(csv, 'wb') do |file|
42
+ file.write(params[:csv_import][:csv_file].read)
43
43
  end
44
44
  check_paths csv
45
45
  if @path_list.present?
@@ -48,13 +48,13 @@ module CdmMigrator
48
48
  parse_csv(csv, params[:csv_import][:mvs])
49
49
 
50
50
  ingest = BatchIngest.new({
51
- data: @works,
52
- size: @works.length,
53
- csv: csv,
54
- admin_set_id: params[:admin_set],
51
+ data: @works,
52
+ size: @works.length,
53
+ csv: csv,
54
+ admin_set_id: params[:admin_set],
55
55
  collection_id: params[:collection],
56
- user_id: current_user.id,
57
- message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"","&quot;")
56
+ user_id: current_user.id,
57
+ message: @path_list.blank? ? nil : @path_list.to_s.gsub("\"", "&quot;")
58
58
  })
59
59
  if ingest.save! && @path_list.blank?
60
60
  BatchCreateWorksJob.perform_later(ingest, current_user)
@@ -75,9 +75,8 @@ module CdmMigrator
75
75
  end
76
76
 
77
77
  def generate
78
- headers = ['type','url']
79
- skip = ["id", "head", "tail", "depositor", "date_uploaded", "date_modified", "import_url", "thumbnail_id",
80
- "embargo_id", "lease_id", "access_control_id", "representative_id"]
78
+ headers = %w(type url)
79
+ skip = %w(id head tail depositor date_uploaded date_modified import_url thumbnail_id embargo_id lease_id access_control_id representative_id)
81
80
  GenericWork.new.attributes.each do |key, val|
82
81
  headers << "work_#{key}" unless skip.include? key
83
82
  end
@@ -87,19 +86,108 @@ module CdmMigrator
87
86
  fname = "template_#{DateTime.now.to_i}"
88
87
  render plain: CSV.generate { |csv| csv << headers }, content_type: 'text/csv'
89
88
  end
90
-
91
- private
89
+
90
+ def edit
91
+ @collections = ::Collection.all.map { |c| [c.title.first, c.id] }
92
+ end
93
+
94
+ def update
95
+ mvs = params[:csv_update][:mvs]
96
+ csv = CSV.parse(params[:csv_update][:csv_file].read, headers: true, encoding: 'utf-8').map(&:to_hash)
97
+ csv.each do |row|
98
+ obj = ActiveFedora::Base.find row['id']
99
+ type = row.first.last
100
+ if type.nil?
101
+ next
102
+ elsif type.include? "Work"
103
+ metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
104
+ elsif type.include? "File"
105
+ metadata = create_data(row.except('id', 'type'), work_form(type), obj, mvs)
106
+ end
107
+ unless metadata.nil?
108
+ obj.attributes = metadata
109
+ obj.save
110
+ end
111
+ end
112
+ flash[:notice] = "csv successfully uploaded"
113
+ redirect_to csv_edit_path
114
+ end
115
+
116
+ def export
117
+ solr = RSolr.connect url: Account.find_by(tenant: Apartment::Tenant.current).solr_endpoint.url
118
+ response = solr.get 'select', params: {
119
+ q: "member_of_collection_ids_ssim:#{params[:collection_id]}",
120
+ fl: "id"
121
+ }
122
+ unless response['response']['docs'].empty? || response['response']['docs'][0].empty?
123
+ work_ids = response['response']['docs'].map { |doc| doc['id'] }
124
+ end
125
+ #works = ::ActiveFedora::Base.where member_of_collection_ids_ssim: params[:collection_id]
126
+ @csv_headers = ['type'] + work_fields
127
+ @csv_array = [@csv_headers.join(',')]
128
+ work_ids.each do |work_id|
129
+ doc = ::SolrDocument.find work_id
130
+ add_line doc
131
+ doc._source[:file_set_ids_ssim].each do |file_id|
132
+ file_doc = ::SolrDocument.find file_id
133
+ add_line file_doc
134
+ end
135
+ end
136
+
137
+ send_data @csv_array.join("\n"),
138
+ :type => 'text/csv; charset=iso-8859-5; header=present',
139
+ :disposition => "attachment; filename=export.csv"
140
+ end
141
+
142
+ private
92
143
 
93
144
  def authenticate
94
145
  authorize! :create, available_works.first
95
146
  end
96
147
 
148
+ def add_line doc
149
+ line_hash = {}
150
+ line_hash['type'] = doc._source[:has_model_ssim].first
151
+ work_fields.each do |field|
152
+ line_hash[field] = create_cell doc, field
153
+ end
154
+ @csv_array << line_hash.values_at(*@csv_headers).map { |cell| cell = '' if cell.nil?; "\"#{cell.gsub("\"", "\"\"")}\"" }.join(',')
155
+
156
+ end
157
+
158
+ def work_fields
159
+ @fields ||= available_works.map { |work| work.new.attributes.keys }.flatten.uniq - excluded_fields
160
+ end
161
+
162
+ def excluded_fields
163
+ %w[date_uploaded date_modified head tail state proxy_depositor on_behalf_of arkivo_checksum label
164
+ relative_path import_url part_of resource_type access_control_id
165
+ representative_id thumbnail_id rendering_ids admin_set_id embargo_id
166
+ lease_id]
167
+ end
168
+
169
+ def create_cell w, field
170
+ if field.include? 'date'
171
+ if w._source[field+'_tesim'].is_a?(Array)
172
+ w._source[field+'_tesim'].join('|')
173
+ else
174
+ w._source[field+'_tesim']
175
+ end
176
+ elsif w.respond_to?(field.to_sym)
177
+ if w.send(field).is_a?(Array)
178
+ w.send(field).join('|')
179
+ else
180
+ w.send(field)
181
+ end
182
+ end
183
+ end
184
+
97
185
  def available_works
98
186
  @available_works ||= Hyrax::QuickClassificationQuery.new(current_user).authorized_models
99
187
  end
100
188
 
101
189
  def parse_csv csv, mvs
102
- csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
190
+ csv = CSV.parse(File.read(csv), headers: true, encoding: 'utf-8').map(&:to_hash)
103
191
  @works = []
104
192
  csv.each do |row|
105
193
  type = row.first.last
@@ -120,7 +208,7 @@ module CdmMigrator
120
208
  @path_list = {}
121
209
 
122
210
  CSV.foreach(csv_file, headers: true, header_converters: :symbol) do |row|
123
- row_number +=1 # Tells user what CSV row the bogus file path is on
211
+ row_number += 1 # Tells user what CSV row the bogus file path is on
124
212
  next if row[:url].nil?
125
213
  file_path = row[:url]
126
214
  unless File.file?(file_path.gsub("file://", ""))
@@ -139,8 +227,8 @@ module CdmMigrator
139
227
 
140
228
  def available_translations
141
229
  {
142
- 'en' => 'English',
143
- 'fr' => 'French'
230
+ 'en' => 'English',
231
+ 'fr' => 'French'
144
232
  }
145
233
  end
146
234
 
@@ -162,12 +250,12 @@ module CdmMigrator
162
250
  end
163
251
 
164
252
  def create_data data, type, object, mvs
165
- final_data = {}
253
+ final_data = {}
166
254
  accepted_terms = type.required_fields + secondary_terms(type)
167
255
  data.each do |key, att|
168
- if(att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym) )
256
+ if (att.nil? || att.empty? || key.to_s.include?("object_type") || !accepted_terms.include?(key.to_sym))
169
257
  next
170
- elsif(object.send(key).nil?)
258
+ elsif (object.send(key).nil?)
171
259
  final_data[key] = att
172
260
  else
173
261
  final_data[key] = att.split(mvs)
@@ -176,23 +264,23 @@ module CdmMigrator
176
264
  final_data
177
265
  end
178
266
 
179
- def create_lease visibility, status_after, date
180
- lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
181
- visibility_after_lease: status_after, lease_expiration_date: @lease_date)
182
- lease.save
183
- end
267
+ def create_lease visibility, status_after, date
268
+ lease = Hydra::AccessControls::Lease.new(visibility_during_lease: visibility,
269
+ visibility_after_lease: status_after, lease_expiration_date: @lease_date)
270
+ lease.save
271
+ end
184
272
 
185
- def create_embargo visibility
186
- embargo = Hydra::AccessControls::Embargo.new
187
- embargo.visibility_during_embargo = visibility
188
- embargo.visibility_after_embargo = @status_after
189
- embargo.embargo_release_date = @embargo_date
190
- embargo.save
191
- end
273
+ def create_embargo visibility
274
+ embargo = Hydra::AccessControls::Embargo.new
275
+ embargo.visibility_during_embargo = visibility
276
+ embargo.visibility_after_embargo = @status_after
277
+ embargo.embargo_release_date = @embargo_date
278
+ embargo.save
279
+ end
192
280
 
193
281
  def log(user)
194
- Hyrax::Operation.create!(user: user,
195
- operation_type: "Attach Remote File")
282
+ Hyrax::Operation.create!(user: user,
283
+ operation_type: "Attach Remote File")
196
284
  end
197
- end
285
+ end
198
286
  end
@@ -0,0 +1,10 @@
1
+ module CdmMigrator
2
+ class UpdateObjectJob < ActiveJob::Base
3
+
4
+
5
+ def perform(attributes)
6
+ obj = ActiveFedora::Base.find
7
+
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,18 @@
1
+
2
+ <h1><span class="fa fa-map"></span> Export </h1>
3
+
4
+ <%= form_tag main_app.csv_export_path, method: :post do %>
5
+ <%= select_tag 'collection_id', options_for_select(@collections), include_blank: true %>
6
+ <%= submit_tag "Download CSV", class: 'btn btn-primary' %>
7
+ <% end %>
8
+
9
+ <h1><span class="fa fa-map"></span> Update </h1>
10
+
11
+ <%= form_for :csv_update, url: csv_update_path do |f| %>
12
+ <%= f.label 'Multi-value Separator:' %>
13
+ <%= f.text_field 'mvs' %>
14
+ <br />
15
+ <%= f.file_field 'csv_file' %>
16
+ <br />
17
+ <%= f.submit 'Save' %>
18
+ <% end %>
@@ -5,6 +5,9 @@ Rails.application.routes.draw do
5
5
  get '/cdm_migrator/my/batches', to: 'cdm_migrator/csv#index', as: 'csv_my_batches'
6
6
  get '/cdm_migrator/batches', to: 'cdm_migrator/csv#index', as: 'csv_all_batches'
7
7
  get '/cdm_migrator/rerun/:id', to: 'cdm_migrator/csv#rerun', as: 'csv_rerun'
8
+ get '/cdm_migrator/edit', to: 'cdm_migrator/csv#edit', as: 'csv_edit'
9
+ post '/cdm_migrator/export', to: 'cdm_migrator/csv#export', as: 'csv_export'
10
+ post '/cdm_migrator/update', to: 'cdm_migrator/csv#update', as: 'csv_update'
8
11
 
9
12
  get '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'file_path_checker'
10
13
  post '/cdm_migrator/file_path_checker', to: 'cdm_migrator/csv#file_path_checker', as: 'check_file_paths'
@@ -1,3 +1,3 @@
1
1
  module CdmMigrator
2
- VERSION = '3.0.3'
2
+ VERSION = '3.1.0'
3
3
  end
@@ -19,6 +19,10 @@
19
19
  <%= menu.nav_link(main_app.csv_my_batches_path) do %>
20
20
  <span class="fa fa-database"></span> <span class="sidebar-action-text"><%= t('Batches') %></span>
21
21
  <% end %>
22
+
23
+ <%= menu.nav_link(main_app.csv_edit_path) do %>
24
+ <span class="fa fa-angle-double-up"></span> <span class="sidebar-action-text"><%= t('Batch Update') %></span>
25
+ <% end %>
22
26
  <% end %>
23
27
  </li>
24
28
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cdm_migrator
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.3
4
+ version: 3.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - sephirothkod
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-02-10 00:00:00.000000000 Z
11
+ date: 2020-05-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -90,6 +90,7 @@ files:
90
90
  - app/jobs/cdm_migrator/batch_create_works_job.rb
91
91
  - app/jobs/cdm_migrator/cdm_ingest_files_job.rb
92
92
  - app/jobs/cdm_migrator/create_work_job.rb
93
+ - app/jobs/cdm_migrator/update_object_job.rb
93
94
  - app/mailers/cdm_migrator/application_mailer.rb
94
95
  - app/models/cdm_migrator/application_record.rb
95
96
  - app/models/cdm_migrator/batch_ingest.rb
@@ -102,6 +103,7 @@ files:
102
103
  - app/views/cdm_migrator/csv/_path_list.html.erb
103
104
  - app/views/cdm_migrator/csv/_results_pagination.html.erb
104
105
  - app/views/cdm_migrator/csv/_tabs.html.erb
106
+ - app/views/cdm_migrator/csv/edit.html.erb
105
107
  - app/views/cdm_migrator/csv/file_path_checker.html.erb
106
108
  - app/views/cdm_migrator/csv/index.html.erb
107
109
  - app/views/cdm_migrator/csv/upload.html.erb