osc_machete_rails 1.2.0

data/lib/generators/osc_machete_rails/scaffold_controller/templates/controller.rb

@@ -0,0 +1,120 @@
+<% if namespaced? -%>
+require_dependency "<%= namespaced_path %>/application_controller"
+
+<% end -%>
+<% module_namespacing do -%>
+class <%= controller_class_name %>Controller < ApplicationController
+  before_action :set_<%= singular_table_name %>, only: [:show, :edit, :update, :destroy, :submit, :copy]
+
+  # GET <%= route_url %>
+  # GET <%= route_url %>.json
+  def index
+    @<%= plural_table_name %> = <%= "#{class_name}.preload(:#{job.plural_name})" %>
+  end
+
+  # GET <%= route_url %>/1
+  # GET <%= route_url %>/1.json
+  def show
+  end
+
+  # GET <%= route_url %>/new
+  def new
+    @<%= singular_table_name %> = <%= orm_class.build(class_name) %>
+  end
+
+  # GET <%= route_url %>/1/edit
+  def edit
+  end
+
+  # POST <%= route_url %>
+  # POST <%= route_url %>.json
+  def create
+    @<%= singular_table_name %> = <%= orm_class.build(class_name, "#{singular_table_name}_params") %>
+
+    respond_to do |format|
+      if @<%= orm_instance.save %>
+        format.html { redirect_to @<%= singular_table_name %>, notice: <%= "'#{human_name} was successfully created.'" %> }
+        format.json { render :show, status: :created, location: @<%= singular_table_name %> }
+      else
+        format.html { render :new }
+        format.json { render json: @<%= orm_instance.errors %>, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # PATCH/PUT <%= route_url %>/1
+  # PATCH/PUT <%= route_url %>/1.json
+  def update
+    respond_to do |format|
+      if @<%= orm_instance.update("#{singular_table_name}_params") %>
+        format.html { redirect_to @<%= singular_table_name %>, notice: <%= "'#{human_name} was successfully updated.'" %> }
+        format.json { render :show, status: :ok, location: @<%= singular_table_name %> }
+      else
+        format.html { render :edit }
+        format.json { render json: @<%= orm_instance.errors %>, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  # DELETE <%= route_url %>/1
+  # DELETE <%= route_url %>/1.json
+  def destroy
+    respond_to do |format|
+      if @<%= orm_instance.destroy %>
+        format.html { redirect_to <%= index_helper %>_url, notice: <%= "'#{human_name} was successfully destroyed.'" %> }
+        format.json { head :no_content }
+      else
+        format.html { redirect_to <%= index_helper %>_url, alert: <%= "\"#{human_name} failed to be destroyed: \#{@#{orm_instance.errors}.to_a}\"" %> }
+        format.json { render json: @<%= orm_instance.errors %>, status: :internal_server_error }
+      end
+    end
+  end
+
+  # PUT <%= route_url %>/1/submit
+  # PUT <%= route_url %>/1/submit.json
+  def submit
+    respond_to do |format|
+      if @<%= singular_table_name %>.submitted?
+        format.html { redirect_to <%= index_helper %>_url, alert: <%= "'#{human_name} has already been submitted.'" %> }
+        format.json { head :no_content }
+      elsif @<%= singular_table_name %>.submit
+        format.html { redirect_to <%= index_helper %>_url, notice: <%= "'#{human_name} was successfully submitted.'" %> }
+        format.json { head :no_content }
+      else
+        format.html { redirect_to <%= index_helper %>_url, alert: <%= "\"#{human_name} failed to be submitted: \#{@#{orm_instance.errors}.to_a}\"" %> }
+        format.json { render json: @<%= orm_instance.errors %>, status: :internal_server_error }
+      end
+    end
+  end
+
+  # PUT <%= route_url %>/1/copy
+  def copy
+    @<%= singular_table_name %> = @<%= singular_table_name %>.copy
+
+    respond_to do |format|
+      if @<%= orm_instance.save %>
+        format.html { redirect_to @<%= singular_table_name %>, notice: <%= "'#{human_name} was successfully copied.'" %> }
+        format.json { render :show, status: :created, location: @<%= singular_table_name %> }
+      else
+        format.html { redirect_to <%= index_helper %>_url, alert: <%= "\"#{human_name} failed to be copied: \#{@#{orm_instance.errors}.to_a}\"" %> }
+        format.json { render json: @<%= orm_instance.errors %>, status: :unprocessable_entity }
+      end
+    end
+  end
+
+  private
+    # Use callbacks to share common setup or constraints between actions.
+    def set_<%= singular_table_name %>
+      @<%= singular_table_name %> = <%= orm_class.find("#{class_name}.preload(:#{job.plural_name})", "params[:id]") %>
+    end
+
+    # Only allow a trusted parameter "white list" through.
+    def <%= "#{singular_table_name}_params" %>
+      <%- if attributes_names.empty? -%>
+      params.fetch(:<%= singular_table_name %>, {})
+      <%- else -%>
+      params.require(:<%= singular_table_name %>).permit!
+      <%- end -%>
+    end
+end
+<% end -%>

data/lib/generators/osc_machete_rails/scaffold_generator.rb

@@ -0,0 +1,35 @@
+require 'rails/generators/rails/scaffold/scaffold_generator'
+
+class OscMacheteRails::ScaffoldGenerator < Rails::Generators::ScaffoldGenerator
+  source_root File.expand_path('../templates', __FILE__)
+
+  attr_reader :orig_args
+
+  def initialize(args, *options)
+    @orig_args = args
+    super
+  end
+
+  # override ModelGenerator
+  remove_hook_for :orm
+
+  # hook for workflow model/migration
+  hook_for :workflow_model, type: :boolean
+
+  # hook for job model/migration
+  hook_for :job_model, type: :boolean do |model|
+    invoke model, %W[#{singular_table_name}_job #{singular_table_name}:references]
+  end
+
+  # override hook for adding config/routes
+  hook_for :resource_route, required: true
+
+  # hook for workflow batch script template
+  hook_for :workflow_template, type: :boolean
+
+  # override ScaffoldGenerator
+  hook_for :scaffold_controller, required: true
+
+  # remove scaffold_stylesheet
+  remove_hook_for :stylesheet_engine
+end

data/lib/generators/osc_machete_rails/workflow_model/USAGE

@@ -0,0 +1,38 @@
+Description:
+    Stubs out a new workflow model. Pass the model name, either CamelCased or
+    under_scored, and an optional list of attribute pairs as arguments.
+
+    Attribute pairs are field:type arguments specifying the
+    model's attributes. Timestamps are added by default, so you don't have to
+    specify them by hand as 'created_at:datetime updated_at:datetime'.
+
+    You don't have to think up every attribute up front, but it helps to
+    sketch out a few so you can start working with the model immediately.
+
+    This generator invokes your configured ORM and test framework, which
+    defaults to ActiveRecord and TestUnit.
+
+Available field types:
+
+    For more information see:
+
+        `rails generate model --help`
+
+Example:
+    `rails generate osc_machete_rails:workflow_model Thing`
+
+        For ActiveRecord and TestUnit it creates:
+            Model:      app/models/thing.rb
+            Test:       test/models/thing_test.rb
+            Fixtures:   test/fixtures/things.yml
+            Migration:  db/migrate/XXX_create_things.rb
+
+        Note:
+            It will default to using the Job model for its Machete jobs.
+
+    `rails generate osc_machete_rails:workflow_model Container
+        name:string pressure:decimal container_job:jobs`
+
+        Note:
+            This will do all of above as well as set the ContainerJob as its
+            Machete jobs.

data/lib/generators/osc_machete_rails/workflow_model/workflow_model_generator.rb

@@ -0,0 +1,12 @@
+class OscMacheteRails::WorkflowModelGenerator < Rails::Generators::NamedBase
+  source_root File.expand_path("../templates", __FILE__)
+
+  def initialize(args, *options)
+    args |= %w(staged_dir:string)
+
+    super
+  end
+
+  argument :attributes, type: :array, default: [], banner: "field[:type][:index] field[:type][:index]"
+  hook_for :orm, required: true
+end

data/lib/generators/osc_machete_rails/workflow_template/USAGE

@@ -0,0 +1,19 @@
+Description:
+    Stubs out a new workflow template. Pass the template name, either
+    CamelCased or under_scored, and an optional list of attribute pairs as
+    arguments.
+
+    Attribute pairs are field:type arguments specifying the
+    model's attributes. Timestamps are added by default, so you don't have to
+    specify them by hand as 'created_at:datetime updated_at:datetime'.
+
+    You don't have to think up every attribute up front, but it helps to
+    sketch out a few so you can start working with the model immediately.
+
+    This generates a batch script in jobs/.
+
+Example:
+    `rails generate osc_machete_rails:workflow_template Thing pressure:decimal`
+
+    Creates:
+        Template:   jobs/thing/main.sh.mustache

data/lib/generators/osc_machete_rails/workflow_template/templates/main.sh.mustache

@@ -0,0 +1,18 @@
+#PBS -N <%= file_name %>
+#PBS -l nodes=1:ppn=12
+#PBS -l walltime=01:00:00
+#PBS -q @oak-batch.osc.edu
+#PBS -j oe
+#PBS -S /bin/bash
+
+echo "---Job started at:"
+date
+
+<% attributes.each do |attribute| -%>
+echo "<%= attribute.name %> = {{{<%= attribute.name %>}}}"
+<% end -%>
+
+sleep 30
+
+echo "---Job finished at:"
+date

data/lib/generators/osc_machete_rails/workflow_template/workflow_template_generator.rb

@@ -0,0 +1,9 @@
+class OscMacheteRails::WorkflowTemplateGenerator < Rails::Generators::NamedBase
+  source_root File.expand_path('../templates', __FILE__)
+
+  argument :attributes, type: :array, default: [], banner: "field[:type][:index] field[:type][:index]"
+
+  def copy_batch_script
+    template "main.sh.mustache", "jobs/#{file_name}/main.sh.mustache"
+  end
+end

data/lib/osc_machete_rails/engine.rb

@@ -0,0 +1,24 @@
+module OscMacheteRails
+  class Engine < ::Rails::Engine
+    isolate_namespace OscMacheteRails
+
+    config.app_generators do |g|
+      g.workflow_model      true
+      g.job_model           true
+      g.workflow_template   true
+    end
+
+    config.before_initialize do
+      OscMacheteRails.update_status_of_all_active_jobs_on_each_request = true
+    end
+
+    config.after_initialize do
+      if OscMacheteRails.update_status_of_all_active_jobs_on_each_request
+        # set before action on both Engine controllers and main App controllers
+        # to update the status of all the active jobs
+        ::ApplicationController.before_action -> { OscMacheteRails::Statusable.update_status_of_all_active_jobs }
+        ApplicationController.before_action -> { OscMacheteRails::Statusable.update_status_of_all_active_jobs }
+      end
+    end
+  end
+end

data/lib/osc_machete_rails/helper.rb

@@ -0,0 +1,25 @@
+module OscMacheteRails
+  module Helper
+    def status_label(job, tag = :span)
+      job ||= OpenStruct.new status: OSC::Machete::Status.not_submitted
+      text = job.status.to_s
+
+      label_class = 'label-default'
+      if job.failed?
+        label_class = 'label-danger'
+      elsif job.passed?
+        label_class = 'label-success'
+        text = "Completed"
+      elsif job.active?
+        label_class = 'label-primary'
+      end
+
+      content_tag tag, class: %I(status-label label #{label_class}) do
+        text
+      end
+    end
+    alias_method :job_status_label, :status_label
+  end
+end
+
+ActionView::Base.send :include, OscMacheteRails::Helper

data/lib/osc_machete_rails/statusable.rb

@@ -0,0 +1,199 @@
+module OscMacheteRails
+  # Methods that deal with pbs batch job status management
+  # within a Rails ActiveRecord model
+  module Statusable
+
+
+    delegate :not_submitted?, :submitted?, :completed?, :passed?, :failed?, :active?, to: :status
+
+    def status=(s)
+      super(OSC::Machete::Status.new(s).char)
+    end
+
+    # getter returns a Status value from CHAR or a Status value
+    def status
+      OSC::Machete::Status.new(super)
+    end
+
+    # delete the batch job and update status
+    # may raise PBS::Error as it is unhandled here!
+    def stop(update: true)
+      return unless status.active?
+
+      job.delete
+      update(status: OSC::Machete::Status.failed) if update
+    end
+
+    def self.included(obj)
+      # track the classes that include this module
+      self.classes << Kernel.const_get(obj.name) unless obj.name.nil?
+
+      # add class methods
+      obj.send(:extend, ClassMethods)
+
+      begin
+        obj.class_eval do
+          # Store job object info in a JSON column and replace old column methods
+          if respond_to?(:table_exists?) && table_exists? && respond_to?(:column_names) && column_names.include?('job_cache')
+            store :job_cache, accessors: [ :script, :pbsid, :host ], coder: JSON
+            delegate :script_name, to: :job
+            define_method :job_path do
+              job.path
+            end
+          else
+            define_method(:job_cache) do
+              {
+                script: (job_path && script_name) ? Pathname.new(job_path).join(script_name) : nil,
+                pbsid: pbsid,
+                host: nil
+              }
+            end
+          end
+        end
+      rescue StandardError => e
+        msg = "\nIf you are precompiling assets, you can ignore this:\n\tError thrown: #{e.inspect}"
+        msg += "\n\tError thrown in OscMacheteRails::Statusable.included when trying to add job_cache to the class including Statusable: #{obj.name}."
+        msg += "\n\tjob_cache method was not added to class." unless obj.method_defined? :job_cache
+        msg += "\n\n"
+        STDERR.puts msg
+      end
+    end
+
+
+    def self.classes
+      @classes ||= []
+    end
+
+    # for each Statusable, call update_status! on active jobs
+    def self.update_status_of_all_active_jobs
+      # to eager load classes, set config.eager_load to true or execute Rails.application.eager_load!
+      if self.classes.empty?
+        Rails.logger.warn "Statusable.classes Array is empty. This should contain a list of all the classes that include Statusable. " \
+          "This could occur if the Rails app is not configured to eager load classes."
+      else
+        Rails.logger.debug "Updating active job statuses via Statusable.update_status_of_all_active_jobs."
+      end
+
+      self.classes.each do |klass|
+        klass.active.to_a.each(&:update_status!) if klass.respond_to?(:active)
+      end
+    end
+
+    # class methods to extend a model with
+    module ClassMethods
+      # scope to get all of the jobs that are in an active state
+      # or have a pbsid
+      def active
+        # FIXME: what about OR i.e. where
+        #
+        #     status in active_values OR (pbsid != null and status == null)
+        #
+        # will need to use STRING for the sql instead of this.
+        where(status: OSC::Machete::Status.active_values.map(&:char))
+      end
+    end
+
+    # Setter that accepts an OSC::Machete::Job instance
+    #
+    # @param [Job] new_job The Job object to be assigned to the Statusable instance.
+    def job=(new_job)
+      if self.has_attribute?(:job_cache)
+        self.script = new_job.script_path.to_s
+        self.pbsid = new_job.pbsid
+        self.host = new_job.host if new_job.respond_to?(:host)
+      else
+        self.script_name = new_job.script_name
+        self.job_path = new_job.path.to_s
+        self.pbsid = new_job.pbsid
+      end
+
+      begin
+        self.status = new_job.status
+      rescue PBS::Error => e
+        # a safe default
+        self.status = OSC::Machete::Status.queued
+
+        # log the error
+        Rails.logger.error("After submitting the job with pbsid: #{pbsid}," \
+                           " checking the status raised a PBS::Error: #{e.message}")
+      end
+    end
+
+    # Returns associated OSC::Machete::Job instance
+    def job
+      OSC::Machete::Job.new(job_cache.symbolize_keys)
+    end
+
+    # Build the results validation method name from script_name attr
+    # using ActiveSupport methods
+    #
+    # Call this using the Rails console to see what method you should implement
+    # to support results validation for that job.
+    #
+    # @return [String] A string representing a validation method name from script_name attr
+    # using ActiveSupport methods
+    def results_validation_method_name
+      File.basename(script_name, ".*").underscore.parameterize('_') + "_results_valid?"
+    end
+
+    # A hook that can be overidden with custom code
+    # also looks for default validation methods for existing
+    # WARNING: THIS USES ActiveSupport::Inflector methods underscore and parameterize
+    #
+    # @return [Boolean] true if the results script is present
+    def results_valid?
+      valid = true
+
+      if self.respond_to? :script_name && !script_name.nil?
+        if self.respond_to?(results_validation_method_name)
+          valid = self.send(results_validation_method_name)
+        end
+      end
+
+      valid
+    end
+
+    #FIXME: should have a unit test for this!
+    # job.update_status! will update and save object
+    # if submitted? and ! completed? and status changed from previous state
+    # force will cause status to update regardless of completion status,
+    # redoing the validations. This way, if you are fixing validation methods
+    # you can use the Rails console to update the status of a Workflow by doing this:
+    #
+    #     Container.last.jobs.each {|j| j.update_status!(force: true) }
+    #
+    # Or for a single statusable such as job:
+    #
+    #     job.update_status!(force: true)
+    #
+    # FIXME: should log whether a validation method was called or
+    # throw a warning that no validation method was found (the one that would have been called)
+    #
+    # @param [Boolean, nil] force Force the update. (Default: false)
+    def update_status!(force: false)
+      # this will make it easier to differentiate from current_status
+      cached_status = status
+
+      # by default only update if its an active job
+      if  (cached_status.not_submitted? && pbsid) || cached_status.active? || force
+        # get the current status from the system
+        current_status = job.status
+
+        # if job is done, lets re-validate
+        if current_status.completed?
+          current_status = results_valid? ? OSC::Machete::Status.passed : OSC::Machete::Status.failed
+        end
+
+        if (current_status != cached_status) || force
+          self.status = current_status
+          self.save
+        end
+      end
+
+    rescue PBS::Error => e
+      # we log the error but we just don't update the status
+      Rails.logger.error("During update_status! call on job with pbsid #{pbsid} and id #{id}" \
+                          " a PBS::Error was thrown: #{e.message}")
+    end
+  end
+end

data/lib/osc_machete_rails/version.rb

@@ -0,0 +1,3 @@
+module OscMacheteRails
+  VERSION = "1.2.0"
+end