bulk_ops 0.1.3

data/app/views/bulk_ops/operations/_update_draft_work_search.html.erb

@@ -0,0 +1,59 @@
+<div id="add-works" >
+    <h3>Find Works to add to update</h3>
+    
+    <form id="ajax-work-search">
+        <div>    
+            <label for="collection-id"> Collection </label>
+            <select name="collection_id" id="collection-id"><%= options_for_select @collections.insert(0,["Any",""]) %> </select>
+        </div>
+        <div>    
+            <label for="admin-set-id"> Administrative Set </label>
+            <select name="admin_set_id" id="admin-set-id"> <%= options_for_select @admin_sets.insert(0,["Any",""]) %> </select>
+        </div>
+        <div>    
+            <label for="workflow_state"> Workflow Step </label>
+            <select name="workflow_state" id="workflow-state"> <%= options_for_select @workflow_states.insert(0,["Any",""]) %></select>
+        </div>
+        
+        <div>
+            <label for="keyword"> keyword search </label>
+            <input id="keyword" type="text" name="q" />
+        </div>
+    </form>
+    <button id="add-works-search" type="button"> Search </button>
+
+    <div id="search-summary">
+        <%= form_tag("#{@operation.id}/edit") do %>
+            <div id="draft-search-controls">
+                <div id="all-results" style="display:none">
+                    <p id="count"></p>
+                    <input type="checkbox" name="add_all_results" id="add-all-results" value="true"/>
+                    <label for="add-all-results">Add all search results to update</label>
+                </div>
+                <input type="submit"  id="draft-update-add-works" class="show-with-results" value="Add to Update" />
+            </div>
+            <div id="draft-search-display" class="show-with-results">
+                <h4>Sample Results (first 15 results)</h4>
+                <button type="button" class="selections select-all">Select All</button>
+                <button type="button" class="selections select-none">Select None</button>
+                <ul id="search-sample">
+                    <li id="template" class="hidden">
+                        <input type="checkbox" name="added_work_ids[]" value=""/>
+                        <%= image_tag("", height: '200', class: 'work-thumb') %>
+                        <div class="work-meta">
+                            <div class="work-title"></div>
+                            <div class="work-desription"></div>
+                        </div>
+                    </li>
+                </ul>
+            </div>
+            <input class="prev-search-field" id="prev-collection-id" type="hidden" name="collection_id" value=""/>
+            <input class="prev-search-field" id="prev-admin-set-id" type="hidden" name="admin_set_id" value=""/>
+            <input class="prev-search-field" id="prev-workflow-state" type="hidden" name="workflow_state" value=""/>
+            <input class="prev-search-field" id="prev-keyword" type="hidden" name="keyword" value=""/>
+        <% end %>
+    </div>
+
+
+</div>
+

data/app/views/bulk_ops/operations/_update_options.html.erb

@@ -0,0 +1,9 @@
+<div id="update-options" class="options-panel bulk-ops-options">
+    <div>
+        <label for="file-method" data-toggle="tooltip" title="These options determine how the update handles files attached to updated objects. The default option leaves all files alone unless you include a column in your spreadsheet to add or remove specific files.">File Update Method:</label>
+        <select name="file_method" id="file-method">
+            <%= options_for_select @file_update_options, @operation.options['file_method'] || "remove-and-add" %> 
+        </select>
+    </div>
+
+</div>

data/app/views/bulk_ops/operations/index.html.erb

@@ -0,0 +1,51 @@
+<% provide :page_title, "List Bulk Operations" %>
+<%= javascript_include_tag "bulk_ops" %>
+<%= stylesheet_link_tag "bulk_ops" %>
+<%= render "bulk_ops_header", title: "Manage Bulk Ingests & Updates" %>
+<%= render "bulk_ops/github_auth_widget" %>
+
+<%= form_tag("destroy_multiple") do %>
+    <div id="bulk-ops-index-header">
+        <input type="checkbox" id="bulk-ops-select-all" />
+        <button id="bulk-ops-delete-group">Delete Selected</button>
+        <a href="new" >
+            <button type="button" id="bulk-ops-new"> New Bulk Operation</button>
+        </a>
+    </div>
+
+    <ul id="operations-index">
+        <% if @active_operations.empty? %>
+            No bulk operations have been created yet.
+        <% end %>
+        <% @active_operations.each do |operation| %>
+            <li>
+                <input type="checkbox" name="operation_ids[]" class="bulk-ops-index-checkbox" value="<%= operation.id %>" />
+                <div class="op-name">
+                    <%= link_to(operation.name,operation)%>
+                </div>
+
+                <div class="op-summary">
+                    <div class="op-type">
+                        <label> Type: </label>
+                        <%=operation.type%>
+                    </div>
+                    <div class="op-stage">
+                        <label> Stage: </label>
+                        <%=operation.stage%>
+                    </div>
+                </div>
+                
+                <div class="op-status">
+                    <label> Status: </label>
+                    <%=operation.status%>
+                        </label>
+                </div>
+                <div class="op-message">
+                    <label> Message: </label>
+                    <%=operation.message%>
+                </div>
+                
+            </li>
+        <% end %>
+    </ul>
+<% end %>

data/app/views/bulk_ops/operations/new.html.erb

@@ -0,0 +1,36 @@
+<%= javascript_include_tag "bulk_ops" %>
+<%= stylesheet_link_tag "bulk_ops" %>
+<% provide :page_title, "New Bulk Operations" %>
+<%= render "bulk_ops_header", title: 'Create New Bulk Ingest or Update' %>
+<%= render "bulk_ops/github_auth_widget" %>
+
+<%= form_tag "/bulk_ops", class: "new-bulk-op", id: "new-bulk-op" do %>
+
+    <div id="operation-name" class="bulk-ops-options">
+        <label data-toggle="tooltip" title="Choose a short descriptive name for this bulk operation" for="bulk-ops-name">New Operation Name:</label>
+        <input type="text" name="name" id="bulk-ops-name" />
+    </div>
+
+    <div class="bulk-ops-options">
+        <label for="op-type" data-toggle="tooltip" title="Would you like to set up a bulk ingest of new works, or a bulk update of existing works?">Ingest or Update?</label>
+        <select name="type"  id="op-type" onchange="$('.bulk-ops-options.options-panel').hide();$('#operation-options').show(); $('#'+this.value+'-options').show()" >
+            <option disabled selected value> -- Choose Operation Type -- </option>
+            <option value="ingest">Ingest </option>
+            <option value="update">Update </option>
+        </select>
+    </div>
+
+    <%= render "operation_options" %>
+    <%= render "ingest_options" %>
+    <%= render "update_options" %>
+
+    <%= render "choose_fields" %>
+
+    <%= render "choose_notifications" %>
+
+    <%= render "git_message" %>
+    
+    <input type="submit" name="create_operation" id="create-operation" value="Create Bulk Operation" />
+
+<% end %>
+

data/app/views/bulk_ops/operations/show.html.erb

@@ -0,0 +1,7 @@
+<% provide :page_title, @operation.name.titleize %>
+<%= javascript_include_tag "embed" %>
+<%= javascript_include_tag "bulk_ops" %>
+<%= stylesheet_link_tag "bulk_ops" %>
+<%= render "bulk_ops_header", title: "Showing Bulk #{@operation.type.capitalize}: #{@operation.name.titleize}" %>
+
+<%= render "show_#{@operation.stage}"%>

data/config/routes.rb

@@ -0,0 +1,25 @@
+BulkOps::Engine.routes.draw do
+
+  get 'bulk_ops/authorize/:user_id', to: 'github_authorization#authorize'
+  post 'bulk_ops/github_logout/:user_id', to: 'github_authorization#logout'
+  post 'bulk_ops/apply', to: 'operations#apply'
+
+  resources :operations, path: :bulk_ops do
+    
+    collection do
+      get :search
+      post :apply
+      post :search
+      post :destroy_multiple
+    end
+
+    member do
+      get :csv
+      post :request_apply
+      post :approve
+      post :edit
+      post :duplicate
+    end
+
+  end
+end

data/db/migrate/20180926190757_create_github_credentials.rb

@@ -0,0 +1,13 @@
+class CreateGithubCredentials < ActiveRecord::Migration[5.0]
+  def change
+    create_table :bulk_ops_github_credentials do |t|
+      t.integer :user_id
+      t.string :username
+      t.string :oauth_token
+      t.string :state
+
+      t.timestamps
+    end
+    add_index :bulk_ops_github_credentials, :user_id
+  end
+end

data/db/migrate/20181017180436_create_bulk_ops_tables.rb

@@ -0,0 +1,40 @@
+class CreateBulkOpsTables < ActiveRecord::Migration[5.0]
+  def change
+
+    create_table :bulk_ops_operations do |t|
+      t.references :user, foreign_key: true
+      t.string :name, null: false, unique: true
+      t.string :stage, null: false
+      t.string :operation_type
+      t.string :commit_sha
+      t.integer :pull_id
+      t.string :status
+      t.text :message
+      t.timestamps
+    end
+
+    create_table :bulk_ops_work_proxies do |t|
+      t.integer :operation_id
+      t.string :work_id
+      t.integer :row_number
+      t.datetime :last_event
+      t.string :status
+      t.text :message
+      t.string :visibility
+      t.string :work_type
+      t.string :reference_identifier
+      t.string :order
+      t.timestamps
+    end
+
+    create_table :bulk_ops_relationships do |t|
+      t.integer :work_proxy_id
+      t.string   :object_identifier
+      t.string   :identifier_type
+      t.string   :relationship_type
+      t.string   :status
+      t.timestamps
+    end
+
+  end
+end

data/lib/bulk_ops.rb

@@ -0,0 +1,15 @@
+require "bulk_ops/version"
+
+module BulkOps
+  dirstring = File.join( File.dirname(__FILE__), 'bulk_ops/**/*.rb')
+  Dir[dirstring].each  do |file| 
+    begin
+      require file 
+    rescue Exception => e
+      puts "ERROR LOADING #{File.basename(file)}: #{e}"
+    end
+  end
+#  require 'bulk_ops/verification'
+#  require 'bulk_ops/verification'
+#  require 'bulk_ops/work_proxy'
+end

data/lib/bulk_ops/create_spreadsheet_job.rb

@@ -0,0 +1,43 @@
+class BulkOps::CreateSpreadsheetJob < ActiveJob::Base
+  
+  queue_as :default
+
+  def perform(branch_name, work_ids, fields, user)
+    csv_file = Tempfile.new('bulk_ops_metadata')
+    csv_file.write(fields.join(','))
+    work_ids.each do |work_id| 
+      if work_csv = work_to_csv(work_id,fields)
+        csv_file.write("\r\n" + work_csv)
+      end
+    end
+    csv_file.close
+    BulkOps::GithubAccess.new(branch_name, user).add_new_spreadsheet(csv_file.path)
+    csv_file.unlink
+  end
+
+  private
+
+  def work_to_csv work_id, fields
+    return false if work_id.empty?
+    begin
+      work = Work.find(work_id)
+    rescue ActiveFedora::ObjectNotFoundError
+      return false
+    end
+    line = ''
+    fields.map do |field_name| 
+      label = false
+      if field_name.downcase.include? "label"
+        label = true
+        field_name = field_name[0..-7]
+      end
+      values = work.send(field_name)
+      values.map do |value|
+        next if value.is_a? DateTime 
+        value = (label ? WorkIndexer.fetch_remote_label(value.id) : value.id) unless value.is_a? String
+        value.gsub("\"","\"\"")
+      end.join(BulkOps::WorkProxy::SEPARATOR).prepend('"').concat('"')
+    end.join(',')
+  end
+
+end

data/lib/bulk_ops/create_work_job.rb

@@ -0,0 +1,14 @@
+#require 'hydra/access_controls'
+#require 'hyrax/workflow/activate_object'
+
+require 'bulk_ops/work_job'
+
+class BulkOps::CreateWorkJob < BulkOps::WorkJob
+
+  private
+
+  def type
+    :create
+  end
+
+end

data/lib/bulk_ops/delete_file_set_job.rb

@@ -0,0 +1,15 @@
+#require 'hydra/access_controls'
+#require 'hyrax/workflow/activate_object'
+
+class BulkOps::DeleteFileSetJob < ActiveJob::Base
+
+  queue_as :ingest
+
+  def perform(file_set_id,user_email)
+    user = User.find_by_email(user_email)
+    Hyrax::Actors::FileSetActor.new(@file_set, user).destroy
+  end
+
+  private
+
+end

data/lib/bulk_ops/engine.rb

@@ -0,0 +1,6 @@
+module BulkOps
+  class Engine < ::Rails::Engine
+    isolate_namespace BulkOps
+    engine_name 'bulk_ops'
+  end
+end

data/lib/bulk_ops/error.rb

@@ -0,0 +1,141 @@
+class BulkOps::Error
+  attr_accessor :type, :row_number, :object_id, :message, :option_name, :file, :option_values, :field, :url
+
+  MAX_ERROR = 50
+
+  def initialize type:, row_number: nil, object_id: nil, message: nil, options_name: nil, option_values: nil, field: nil, url: nil , file: nil
+    @type = type
+    @row_number = row_number
+    @object_id = object_id
+    @message = message
+    @option_name = option_name
+    @option_values = option_values
+    @field = field
+    @file = file
+    @url = url
+  end
+
+  def self.write_errors! errors, git
+    return false if errors.blank?
+    error_file_name = "error_log_#{DateTime.now.strftime("%F-%H%M%p")}.log"
+
+    error_hash = {}
+
+    errors.sort!{|x,y| x.type <=> y.type}
+    error_types = errors.map{|error| error.type}.uniq
+
+    #write errors to error file
+    error_file = Tempfile.new(error_file_name)
+    error_types.each do |error_type|
+      typed_errors = errors.select{|er| er.type == error_type}
+      next if typed_errors.blank?
+      message = self.error_message(error_type, typed_errors)
+      puts "Error message: #{message}"
+      error_file.write(message)
+    end
+    error_file.close
+    git.add_file error_file.path, File.join("errors", error_file_name)
+    error_file.unlink
+    return error_file_name
+  end
+
+  def self.error_message type, errors
+    case type
+    when :mismatched_auth_terms
+      message = "\n-- Controlled Authority IDs and Labels don't match -- \n"
+      message += "The operation is set to create an error if the provided URLs for controlled authority terms do not resolve to the provided labels.\n"
+      if errors.count < MAX_ERROR
+        message += "The following rows were affected:\n"
+        message += errors.map{|error| error.row_number}.join(",")+"\n"
+      else
+        message += "#{errors.count} rows were affected. An example is row # #{errors.first.row_number}.\n"
+      end
+    when :upload_error
+      message = "\n-- Errors uploading files -- \n"
+      message += "Your files looked ok when we checked earlier, but we couldn't access them when we were trying to actually start the operation.\n"
+      if errors.count < MAX_ERROR
+        message += "The following files were affected:\n"
+        message += errors.map{|error| "Row #{row_number}, Filename: #{error.file}"}.join("\n")+"\n" 
+      else
+        message += "#{errors.count} rows were affected. An example is row # #{errors.first.row_number} with file #{errors.first.file}.\n"
+      end
+
+    when :no_work_id_field
+      message = "\n-- Cannot find work id field in spreadsheet -- \n"
+      message += "We were trying to start your operation, but could find find the work id for #{errors.count} different rows of the spreadsheet.\n"
+      message += "Check your spreadsheet and try again.\n"
+      message += errors.map{|arg| "  #{error.object_id || 'new work'}: #{error.message}"}.join("\n") + "\n"
+    when :job_failure
+      message = "\n-- Jobs Failed -- \n:"
+      message += errors.map{|arg| "Error message operating on #{error.object_id || 'new work'}: #{error.message}"}.join("\n") + "\n"
+    when :missing_required_option 
+      message = "\n-- Errors in configuration file -- \nMissing required option(s):"
+      message += errors.map{|arg| error.option_name}.join(", ") + "\n"
+
+    when :invalid_config_value 
+      message = "\n-- Errors in configuration file values --\n" 
+      errors.each do |error|
+        message += "Unacceptable value for #{error.option_name}. Acceptable values include: #{error.option_values}\n"
+      end
+
+    when :cannot_get_headers 
+      message += "\n-- Error Retrieving Field Headers --\n"
+      message += "We cannot retrieve the column headers from metadata spreadsheet on github,\nwhich define the fields for the metadata below.\nEither the connection to github is failing, \nor the metadata spreadsheet on this branch is not properly formatted.\n"
+
+    when :bad_header 
+      message =  "\n-- Error interpreting column header(s) --\n"
+      message += "We cannot interpret all of the headers from your metadata spreadsheet. \nSpecifically, the following headers did not make sense to us:\n"
+      message += errors.map{|error| error.field}.join(", ")+"\n"
+
+    when :cannot_retrieve_label 
+      message = "\n-- Errors Retrieving Remote Labels --\n"
+      urls = errors.map{|error| error.url}.uniq
+      if urls.count < MAX_ERROR
+        urls.each do |url|
+          url_errors = errors.select{|er| er.url == url}
+          message +=  "Error retrieving label for remote url #{url}. \nThis url appears in #{url_errors.count} instances in the spreadsheet.\n"
+          message += "The affected rows are listed here:\n"
+          message += url_errors.map{|er| er.row_number}.compact.join('\n')+"\n"
+        end
+      else
+        message += "There were #{urls.count} different URLs in the spreadsheet that we couldn't retrieve labels for,\n making a total of #{errors.count} url related errors.\n These are too many to list, but an example is #{errors.first.url}\n in row #{errors.first.row_number}.\n"
+      end
+
+    when :cannot_retrieve_url
+      message = "\n-- Errors Retrieving Remote URLs --\n"
+      urls = errors.map{|error| error.url}.uniq
+      if urls.count < MAX_ERROR
+        urls.each do |url|
+          url_errors = errors.select{|er| er.url == url}
+          message +=  "Error retrieving URL for remote authority term #{url}. \nThis term appears in #{url_errors.count} instances in the spreadsheet.\n"
+          message += "The affected rows are listed here:\n"
+          message += url_errors.map{|er| er.row_number}.compact.join('\n')+"\n"
+        end
+      else
+        message += "There were #{urls.count} different controlled vocab terms in the spreadsheet that we couldn't retrieve or create URLs for,\n making a total of #{errors.count} controlled term related errors.\n These are too many to list, but an example is #{errors.first.url}\n in row #{errors.first.row_number}.\n"
+      end
+
+    when :bad_object_reference 
+      message = "\n-- Error: bad object reference --\m" 
+      message += "We enountered #{errors.count} problems resolving object references.\n"
+      if errors.count < MAX_ERROR
+         message += "The row numbers with problems were:\n"
+         message += errors.map{|er| "row number #{er.row_number} references the object #{er.object_id}"}.join("\n")
+      else
+         message += "For example, row number #{errors.first.row_number} references an object identified by #{errors.first.object_id}, which we cannot find."
+      end
+              
+    when :cannot_find_file
+      message = "\n-- Missing File Errors --\n "
+      message += "We couldn't find the files listed on #{errors.count} rows.\n"
+      if errors.count < MAX_ERROR
+        message += "Missing filenames:\n"
+        message += errors.map{|er| er.file}.join("\n")
+      else
+        message += "An example of a missing filename is: #{errors.first.file}\n"
+      end
+      
+    end
+    return message
+  end
+end