bulk_ops 0.1.3

data/lib/bulk_ops/github_access.rb

@@ -0,0 +1,284 @@
+require "octokit"
+require "socket"
+require "securerandom"
+require 'base64'
+
+class BulkOps::GithubAccess
+
+  ROW_OFFSET = 2
+  SPREADSHEET_FILENAME = 'metadata.csv'
+  OPTIONS_FILENAME = 'configuration.yml'
+
+  attr_accessor :name
+
+  def self.auth_url user
+    "#{Octokit.web_endpoint}login/oauth/authorize?client_id=#{client_id}&redirect_uri=#{redirect_endpoint(user)}&state=#{state(user)}&scope=repo"
+  end
+  
+  def self.username user
+    return false unless (cred = BulkOps::GithubCredential.find_by(user_id: user))
+    return false unless (token = cred.oauth_token)
+    Octokit::Client.new(access_token: token).user[:login] rescue false
+  end
+  
+  def self.valid_state?(astate,user_id=nil)
+    return (astate === state(user_id))
+  end
+
+  def self.state user_id=nil
+    user_id = current_user.id if user_id.nil?
+    user_id = user_id.id if user_id.is_a? User
+    user_id = user_id.to_i if user_id.is_a? String
+    cred = BulkOps::GithubCredential.find_by(user_id: user_id) || BulkOps::GithubCredential.create({user_id: user_id, state: SecureRandom.hex })
+    cred.state
+  end
+  
+  def self.auth_token user_id=nil
+    user_id = current_user.id if user_id.nil?
+    user_id = user_id.id if user_id.is_a? User
+    user_id = user_id.to_i if user_id.is_a? String
+    return false unless (cred = BulkOps::GithubCredential.find_by(user_id: user_id))
+    return cred.auth_token || false
+  end
+  
+  def self.set_auth_token!(code,user_id=nil)
+    user_id = current_user.id if user_id.nil?
+    user_id = user_id.id if user_id.is_a? User
+    user_id = user_id.to_i if user_id.is_a? String
+    cred = BulkOps::GithubCredential.find_by(user_id: user_id)
+    result = HTTParty.post("https://github.com/login/oauth/access_token", 
+                           body: {client_id: client_id,
+                                  client_secret: client_secret,
+                                  code: code,
+                                  accept: "json"}.to_json, 
+                           headers: { 'Content-Type' => 'application/json', 
+                                      'Accept' => 'application/json'})
+    cred.update(oauth_token: result.parsed_response["access_token"])
+  end
+  
+  def self.client_id
+    YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_id"]
+  end
+
+  def self.client_secret
+    YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_secret"]
+  end
+  
+  def self.webhook_token
+    YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]["client_secret"]
+  end
+  
+  def self.redirect_endpoint user, ssl: false
+    host = Socket.gethostname
+    host = "localhost" if Rails.env.development? or Rails.env.test?
+    protocol = Rails.env.production? ? "https" : "http"
+    protocol = "https" if ssl
+    "#{protocol}://#{host}/bulk_ops/authorize/#{User.first.id}"
+  end
+
+  def self.create_branch! name 
+    self.new(name).create_branch! 
+  end
+
+  def self.add_file name, file_path, file_name = nil, message: false
+    self.new(name).add_file file_path, file_name, message: message
+  end
+
+  def self.add_contents name, file_name, contents, message: false
+    self.new(name).add_contents file_name, contents, message
+  end
+
+  def self.load_options name, branch: nil
+    self.new(name).load_options branch: branch
+  end
+
+  def self.load_metadata branch:, return_headers: false
+    self.new(branch).load_metadata return_headers: return_headers
+  end
+  
+  def self.update_options name, options, message: false
+    self.new(name).update_options options, message: message
+  end
+
+  def self.list_branches
+    self.new.list_branches
+  end
+
+  def self.list_branch_names user
+    self.new.list_branch_names
+  end
+
+  def initialize(newname="dummy", user = nil)
+    @name = newname.parameterize
+    @user = user
+  end
+
+  def create_branch!
+    client.create_ref repo, "heads/#{name}", current_master_commit_sha
+  end
+
+  def delete_branch!
+    return false unless list_branch_names.include? name
+    client.delete_branch repo, name
+  end
+
+  def add_file file_path, file_name = nil, message: nil
+    file_name ||= File.basename(file_path)
+    #    unless (file_name.downcase == "readme.md") || (file_name.downcase.include? "#{name}/")
+    file_name = File.join(name, file_name) unless file_name.downcase.include? "#{name.downcase}/"
+    message ||= "adding file #{file_name} to github branch #{name}"
+    client.create_contents(repo, file_name, message, file: file_path, branch: name)
+  end
+
+  def add_contents file_name, contents, message=false
+    message ||= "adding file #{file_name} to github branch #{name}"
+    begin
+      client.create_contents(repo, file_name, message, contents, branch: name)
+    rescue Octokit::UnprocessableEntity
+      sha = get_file_sha(file_name)
+      client.update_contents(repo, file_name, message, sha, contents, branch: name)
+    end
+  end
+
+  def add_new_spreadsheet file, message=false
+    if file.is_a? Tempfile
+      file.close
+      add_file file.path, SPREADSHEET_FILENAME, message: message
+    elsif file.is_a?(String) && File.file?(file)
+      add_file file, SPREADSHEET_FILENAME, message: message
+    elsif file.is_a? String
+      add_contents(spreadsheet_path, SPREADSHEET_FILENAME, message: message)
+    end
+  end
+
+  def list_branches
+    client.branches(repo).select{|branch| branch[:name] != "master"}
+  end
+
+  def list_branch_names
+    list_branches.map{|branch| branch[:name]}
+  end
+
+  def update_spreadsheet file, message: false
+    message ||= "updating metadata spreadsheet through hyrax browser interface."
+    sha = get_file_sha(spreadsheet_path)
+    file = File.new(file) if file.is_a?(String) && File.exist?(file)
+    client.update_contents(repo, spreadsheet_path, message, sha, file.read, branch: name)
+  end
+
+  def update_options options, message: false
+    if options['reference_column_name']
+      (options['ignored_columns'] ||= []) << options['reference_column_name']
+      (options['reference_identifier'] ||= []) << options['reference_column_name']
+    end
+
+    options['ignored_columns'] =  options['ignored_columns'].reject { |c| c.empty? } if options['ignored_columns'].present?
+
+    message ||= "updating metadata spreadsheet through hyrax browser interface."
+    sha = get_file_sha(options_path)
+    client.update_contents(repo, options_path, message, sha, YAML.dump(options), branch: name)
+  end
+
+  def load_options branch: nil
+    branch ||= name
+    YAML.load(Base64.decode64(get_file_contents(options_path, ref: branch)))
+  end
+
+  def spreadsheet_count branch: nil
+    branch ||= name
+    Base64.decode64(get_file_contents(spreadsheet_path, ref: branch)).lines.count - 1
+  end
+
+  def load_metadata branch: nil, return_headers: false
+    branch ||= name
+    CSV.parse(Base64.decode64(get_file_contents(spreadsheet_path, ref: branch)), {headers: true, return_headers: return_headers})
+  end
+
+  def log_ingest_event log_level, row_number, event_type, message, commit_sha = nil
+    commit_sha ||= current_branch_commit_sha
+    #TODO WRITE THIS CODE
+  end
+
+  def create_pull_request message: false
+    begin
+      message ||= "Apply update #{name} through Hyrax browser interface"
+      pull = client.create_pull_request(repo, "master", name, message)
+      pull["number"]
+    rescue Octokit::UnprocessableEntity
+      return false
+    end
+  end
+
+  def can_merge?
+    return true
+  end
+
+  def merge_pull_request pull_id, message: false
+    client.merge_pull_request(repo, pull_id, message)
+  end
+
+  def get_metadata_row row_number
+    @current_metadata ||= load_metadata
+    @current_metadata[row_number - ROW_OFFSET]
+  end
+  
+  def get_past_metadata_row commit_sha, row_number
+    past_metadata = Base64.decode64( client.contents(repo, path: filename, ref: commit_sha) )
+    past_metadata[row_number - ROW_OFFSET]
+  end
+
+  def get_file filename
+    client.contents(repo, path: filename, ref: name)
+  end
+
+  def get_file_contents filename, ref: nil
+    ref ||= name
+    client.contents(repo, path: filename, ref: ref)[:content]
+  end
+
+  def get_file_sha filename
+    client.contents(repo, path: filename, ref: name)[:sha]
+  end
+
+  def repo
+    github_config["repo"]
+  end
+
+  def spreadsheet_path
+    "#{name}/#{SPREADSHEET_FILENAME}"
+  end
+
+  private
+
+  def options_path
+    "#{name}/#{OPTIONS_FILENAME}"
+  end
+
+  def current_master_commit_sha
+    client.branch(repo,"master").commit.sha
+  end
+
+  def current_branch_commit_sha
+    client.branch(repo, name).commit.sha
+  end
+
+  def client
+    return @client if @client
+    return default_client if @user.nil?
+    return default_client unless (cred = BulkOps::GithubCredential.find_by(user_id: @user.id))
+    return default_client unless (token = cred.oauth_token)
+    client ||= Octokit::Client.new(access_token: token)
+    return default_client unless client.user[:login].is_a? String
+    @client = client
+  end
+
+  def default_client
+    @client ||= Octokit::Client.new(login: github_config["default_user"], password: github_config["default_password"])
+  end
+
+  def github_config
+    @github_config ||=  YAML.load_file("#{Rails.root.to_s}/config/github.yml")[Rails.env]
+  end
+
+
+end

data/lib/bulk_ops/github_credential.rb

@@ -0,0 +1,3 @@
+class BulkOps::GithubCredential < ActiveRecord::Base
+  self.table_name = "bulk_ops_github_credentials"
+end

data/lib/bulk_ops/operation.rb

@@ -0,0 +1,358 @@
+require 'bulk_ops/verification'
+module BulkOps
+  class Operation < ActiveRecord::Base
+    self.table_name = "bulk_ops_operations"
+    belongs_to :user
+    has_many :work_proxies, class_name: "::BulkOps::WorkProxy"
+
+    include BulkOps::Verification
+
+    attr_accessor :work_type, :visibility, :reference_identifier
+    
+    delegate  :can_merge?, :merge_pull_request, to: :git
+
+    INGEST_MEDIA_PATH = "/dams_ingest"
+    TEMPLATE_DIR = "lib/bulk_ops/templates"
+    RELATIONSHIP_COLUMNS = ["parent","child","next"]
+    SPECIAL_COLUMNS = ["parent",
+                       "child",
+                       "order",
+                       "next",
+                       "work_type",
+                       "collection", 
+                       "collection_title",
+                       "collection_id",
+                       "visibility",
+                       "relationship_identifier_type",
+                       "id",
+                       "filename",
+                       "file"]
+    IGNORED_COLUMNS = ["ignore","offline_notes"]
+    OPTION_REQUIREMENTS = {type: {required: true, 
+                                  values:[:ingest,:update]},
+                           file_method: {required: :true,
+                                           values: [:replace_some,:add_remove,:replace_all]},
+                           notifications: {required: true}}
+
+    def self.unique_name name, user
+      while  BulkOps::Operation.find_by(name: name) || BulkOps::GithubAccess.list_branch_names(user).include?(name) do
+        if ['-','_'].include?(name[-2]) && name[-1].to_i > 0
+          name = name[0..-2]+(name[-1].to_i + 1).to_s
+        else
+          name = name + "_1"
+        end
+      end
+      return name
+    end
+
+    def proxy_errors
+      work_proxies.reduce([]) do |errors, proxy| 
+        if proxy.proxy_errors
+          errors += proxy.proxy_errors
+        elsif proxy.status == "job_error"
+          errors += BulkOps::Error.new(type: :job_failure, object_id: proxy.work_id, message: proxy.message)
+        end
+      end
+    end
+
+    def proxy_states
+      states = {}
+      work_proxies.each{|proxy| (states[proxy.status] ||= []) << proxy }
+      states
+    end
+
+    def type
+      operation_type
+    end
+
+    def self.schema
+      ScoobySnacks::METADATA_SCHEMA
+    end
+
+    def schema
+      self.class.schema
+    end
+
+    def work_type
+      options["work_type"] || "work"
+    end
+    
+    def reference_identifier
+      options["reference_identifier"] || "id/row"
+    end
+    
+    def set_stage new_stage
+      update(stage: new_stage)
+    end
+
+    def apply!
+      status = "#{type}ing"
+      update({stage: "running", message: "#{type.titleize} initiated by #{user.name || user.email}"})
+#      @stage = "running"
+      final_spreadsheet
+ 
+# This commented line currently fails because it doesn't pull from the master branch by default
+# It's usually already verified, but maybe we should fix this for double-checking 
+# in the future
+#      return unless verify
+
+      apply_ingest! if ingest?
+      apply_update! if update?
+    end
+
+    def apply_ingest! 
+      #Destroy any existing work proxies (which should not exist for an ingest). Create new proxies from finalized spreadsheet only.
+      work_proxies.each{|proxy| proxy.destroy!}
+
+      #create a work proxy for each row in the spreadsheet
+      @metadata.each_with_index do |values,row_number|
+        next if values.to_s.gsub(',','').blank?
+        work_proxies.create(status: "queued",
+                            last_event: DateTime.now,
+                            row_number: row_number,
+                            message: "created during ingest initiated by #{user.name || user.email}")
+      end
+      # make sure the work proxies we just created are loaded in memory
+      reload
+      #loop through the work proxies to create a job for each work
+      @metadata.each_with_index do |values,row_number|
+        proxy = work_proxies.find_by(row_number: row_number)
+        proxy.update(message: "interpreted at #{DateTime.now.strftime("%d/%m/%Y %H:%M")} " + proxy.message)
+        data = proxy.interpret_data values 
+        next unless proxy.proxy_errors.blank?
+        BulkOps::CreateWorkJob.perform_later(proxy.work_type || "Work",
+                                             user.email,
+                                             data,
+                                             proxy.id,
+                                             proxy.visibility || "open")
+      end
+      # If any errors have occurred, make sure they are logged in github and users are notified.
+      report_errors!
+    end
+
+    def delete_all
+      work_proxies.each do |proxy| 
+        ActiveFedora::Base.find(proxy.work_id).destroy 
+        proxy.update(status: "destroyed", message: "The work created by this proxy was destroyed by the user")
+      end
+    end
+
+    def check_if_finished
+      return unless stage == "running" && !busy?
+      update(stage: accumulated_errors.blank? ? "complete" : "errors" )
+      report_errors!
+      lift_holds
+    end
+
+    def lift_holds
+      work_proxies.each { |proxy| proxy.lift_hold}
+    end
+
+    def place_holds
+      work_proxies.each { |proxy| proxy.place_hold}
+    end
+
+    def apply_update! spreadsheet
+
+      # this array will keep track of any current proxies not included in the final spreadsheet
+      abandoned_proxies = work_proxies.dup
+      # Loop through the final spreadsheet
+      final_spreadsheet.each_with_index do |values,row_number|     
+        # Grab the work id
+        work_id = false
+        values.each{|field,val| work_id = val if ["id","workid","recordid"].include?(field.downcase.gsub(/-_\s/,''))}
+        @operation_errors << BulkOps::Error.new(:no_work_id_field) unless work_id
+
+        #proxy = BulkOps::WorkProxy.find_by(operation_id: id, work_id: values["work_id"])
+        if (proxy = work_proxies.find_by(work_id: work_id))
+          abandoned_proxies.delete(proxy)
+          proxy.update(status: "updating",
+                       row_number: row_number,
+                       message: "update initiated by #{user.name || user.email}")
+        else
+          # Create a proxy for a work that is in the spreadsheet, but wasn't in the initial draft
+          work_proxies.create(status: "queued",
+                              last_event: DateTime.now,
+                              row_number: row_number,
+                              message: "created during update application, which was initiated by #{user.name || user.email}")
+        end
+      end
+
+      # Loop through any proxies in the draft that were dropped from the spreadsheet
+      abandoned_proxies.each do |dead_proxy|
+        dead_proxy.lift_hold
+        dead_proxy.destroy!
+      end
+      
+      #loop through the work proxies to create a job for each work
+      work_proxies.each do |proxy|
+        data = proxy.interpret_data final_spreadsheet[proxy.row_number]
+        BulkOps::UpdateWorkJob.perform_later(proxy.work_type || "",
+                                             user.email,
+                                             data,
+                                             proxy.id,
+                                             proxy.visibility || "private")
+      end
+      report_errors! 
+    end
+
+    def accumulated_errors
+      proxy_errors + (@operation_errors || [])
+      # TODO - make sure this captures all operation errors
+    end
+
+    def report_errors!
+      error_file_name = BulkOps::Error.write_errors!(accumulated_errors, git)
+      notify!(subject: "Errors initializing bulk #{type} in Hycruz", message: "Hycruz encountered some errors while it  was setting up your #{type} and preparing to begin. For most types of errors, the individual rows of the spreadsheet with errors will be ignored and the rest will proceed. Please consult the #{type} summary for real time information on the status of the #{type}. Details about these initialization errors can be seen on Github at the following url: https://github.com/#{git.repo}/blob/#{git.name}/#{git.name}/errors/#{error_file_name}") if error_file_name
+    end
+
+    def create_pull_request message: false
+      return false unless (pull_num = git.create_pull_request(message: message))
+      update(pull_id: pull_num)
+      return pull_num
+    end
+
+    def finalize_draft(fields: nil, work_ids: nil)
+      create_new_spreadsheet(fields: fields, work_ids: work_ids)
+      update(stage: "pending")
+    end
+
+    def create_branch(fields: nil, work_ids: nil, options: nil, operation_type: :ingest)
+      git.create_branch!
+      bulk_ops_dir = Gem::Specification.find_by_name("bulk_ops").gem_dir
+
+      #copy template files
+      Dir["#{bulk_ops_dir}/#{TEMPLATE_DIR}/*"].each do |file| 
+        git.add_file file 
+      end
+
+      #update configuration options 
+      unless options.blank?
+        full_options = YAML.load_file(File.join(bulk_ops_dir,TEMPLATE_DIR, BulkOps::GithubAccess::OPTIONS_FILENAME))
+
+        puts "Creating new bulk_ops options"
+        puts "new options:"
+        puts options.inspect
+        puts "standard options:"
+        puts full_options.inspect
+
+        options.each { |option, value| full_options[option] = value }
+
+        full_options[name] = name
+        full_options[type] = type
+        full_options[status] = status
+        
+        puts "final options:"
+        puts full_options.inspect
+
+        git.update_options full_options
+      end
+
+      create_new_spreadsheet(fields: fields, work_ids: work_ids) if operation_type == :ingest
+    end
+
+    def get_spreadsheet return_headers: false
+      git.load_metadata return_headers: return_headers
+    end
+
+    def spreadsheet_count
+      git.spreadsheet_count
+    end
+
+    def final_spreadsheet
+      @metadata ||= git.load_metadata branch: "master"
+    end
+
+    def update_spreadsheet file, message: nil
+      git.update_spreadsheet(file, message: message)
+    end
+
+    def update_options filename, message=nil
+      git.update_options(filename, message: message)
+    end
+
+    def options
+      return {} if name.nil?
+      return @options if @options
+      branch = running? ? "master" : nil
+      @options ||= git.load_options(branch: branch)
+    end
+
+    def draft?
+      return (stage == 'draft')
+    end
+
+    def running?
+      return (stage == 'running')
+    end
+
+    def complete?
+      return (stage == 'complete')
+    end
+
+    def busy?
+      return true if work_proxies.where(status: "running").count > 0
+      return true if work_proxies.where(status: "queued").count > 0
+      return true if work_proxies.where(status: "starting").count > 0
+      return false
+    end
+
+    def ingest?
+      type == "ingest"
+    end
+
+    def update?
+      type == "update"
+    end
+
+    def delete_branch
+      git.delete_branch!
+    end
+
+    def destroy
+      git.delete_branch!
+      super
+    end
+
+    def self.default_metadata_fields(labels = true)
+      #returns full set of metadata parameters from ScoobySnacks to include in ingest template spreadsheet    
+      field_names = []
+      schema.all_fields.each do |field|
+        field_names << field.name
+        field_names << "#{field.name} Label" if labels && field.controlled?
+      end
+      return field_names
+    end
+
+    def ignored_fields
+      (options['ignored headers'] || []) + IGNORED_COLUMNS
+    end
+
+
+    def error_url
+      "https://github.com/#{git.repo}/tree/#{git.name}/#{git.name}/errors"
+    end
+
+    def filename_prefix
+      @filename_prefix ||= options['filename_prefix']
+    end
+
+    private
+
+    def git
+      @git ||= BulkOps::GithubAccess.new(name, @user)
+    end
+
+    def create_new_spreadsheet(fields: nil, work_ids: nil)
+      work_ids ||= work_proxies.map{|proxy| proxy.work_id}
+      fields ||= self.class.default_metadata_fields
+      work_ids = [] if work_ids.nil?
+      if work_ids.count < 50
+        BulkOps::CreateSpreadsheetJob.perform_now(git.name, work_ids, fields, user)
+      else
+        BulkOps::CreateSpreadsheetJob.perform_later(git.name, work_ids, fields, user)
+      end
+    end
+
+  end
+end