coursemology-evaluator 0.0.0 → 0.1.0

data/Procfile

@@ -0,0 +1 @@
+evaluator: ruby bin/evaluator --host=$HOST --api-token=$API_TOKEN --api-user-email=$API_USER_EMAIL

data/README.md

@@ -1,2 +1,29 @@
-# Coursemology Code Evaluator
-This is the evaluator program which will query Coursemology for pending evaluation jobs.
+# Coursemology Code Evaluator [![Build Status](https://travis-ci.org/Coursemology/evaluator-slave.svg?branch=master)](https://travis-ci.org/Coursemology/evaluator-slave)
+[![Code Climate](https://codeclimate.com/github/Coursemology/evaluator-slave/badges/gpa.svg)](https://codeclimate.com/github/Coursemology/evaluator-slave) [![Coverage Status](https://coveralls.io/repos/Coursemology/evaluator-slave/badge.svg?branch=master&service=github)](https://coveralls.io/github/Coursemology/evaluator-slave?branch=master) [![Security](https://hakiri.io/github/Coursemology/evaluator-slave/master.svg)](https://hakiri.io/github/Coursemology/evaluator-slave/master) [![Inline docs](http://inch-ci.org/github/coursemology/evaluator-slave.svg?branch=master)](http://inch-ci.org/github/coursemology/evaluator-slave)
+
+This is the evaluator program which will query Coursemology for pending evaluation jobs.
+
+## Setting up the Evaluator Slave
+
+### System Requirements
+
+1. Ruby (>= 2.1.0)
+2. Linux (tested on Ubuntu 14.04)
+3. Docker (the user the evaluator runs as must be able to talk to the Docker Remote API endpoint)
+
+### Getting Started
+
+1. Install the gem
+
+   ```sh
+   $ gem install coursemology-evaluator
+   ```
+
+2. Modify `.env` to suit your environment. Point to the host to your Coursemology instance, and 
+   specify the API email and API key.
+
+   1. You might need to configure a new user on your Coursemology instance, enable token 
+      authentication, and grant the `auto_grader` system/instance permission.
+
+3. Start the evaluator using the Procfile. You can use [foreman](https://github.com/ddollar/foreman)
+   or any similar tool to generate system scripts for boot.

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/evaluator

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'coursemology/evaluator'
+
+Coursemology::Evaluator.eager_load!
+Coursemology::Evaluator::CLI.start(ARGV)

data/coursemology-evaluator.gemspec

@@ -13,11 +13,25 @@ Gem::Specification.new do |spec|
   spec.summary       = 'Coursemology programming package evaluator'
   spec.description   = 'Sets up a consistent environment for evaluating programming packages.'
   spec.homepage      = 'http://coursemology.org'
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.files         = `git ls-files -z`.split("\x0").
+                       reject { |f| f.match(/^(test|spec|features)\//) }
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.10'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'factory_girl'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'vcr'
+
+  spec.add_dependency 'activesupport', '~> 4.2.0'
+  spec.add_dependency 'flexirest', '~> 1.2'
+  spec.add_dependency 'faraday_middleware'
+
+  spec.add_dependency 'coursemology-polyglot', '>= 0.0.3'
+  spec.add_dependency 'docker-api', '>= 1.2.5'
+  spec.add_dependency 'rubyzip'
 end

data/lib/coursemology/evaluator.rb

@@ -1,7 +1,35 @@
-require "coursemology/evaluator/version"
+require 'active_support/all'
+require 'flexirest'
+require 'faraday_middleware'
+require 'docker'
+require 'zip'
+Docker.validate_version!
 
-module Coursemology
-  module Evaluator
-    # Your code goes here...
+require 'coursemology/polyglot'
+require 'coursemology/polyglot/extensions'
+require 'coursemology/evaluator/version'
+
+module Coursemology::Evaluator
+  extend ActiveSupport::Autoload
+
+  autoload :Client
+  autoload :DockerContainer
+  autoload :CLI
+  autoload :Models
+  autoload :Services
+  autoload :StringIO
+  autoload :Utils
+
+  eager_autoload do
+    autoload :Logging
+  end
+
+  # The logger to use for the client.
+  mattr_reader(:logger) { ActiveSupport::Logger.new(STDOUT) }
+
+  def self.eager_load!
+    super
+    Coursemology::Polyglot.eager_load!
+    Logging.eager_load!
   end
 end

data/lib/coursemology/evaluator/cli.rb

@@ -0,0 +1,51 @@
+require 'optparse'
+
+class Coursemology::Evaluator::CLI
+  Options = Struct.new(:host, :api_token, :api_user_email, :one_shot)
+
+  def self.start(argv)
+    new.start(argv)
+  end
+
+  def start(argv)
+    run(argv)
+  end
+
+  def run(argv)
+    options = optparse!(argv)
+    Coursemology::Evaluator::Client.initialize(options.host, options.api_user_email,
+                                               options.api_token)
+    Coursemology::Evaluator::Client.new(options.one_shot).run
+  end
+
+  private
+
+  # Parses the options specified on the command line.
+  #
+  # @param [Array<String>] argv The arguments specified on the command line.
+  # @return [Coursemology::Evaluator::CLI::Options]
+  def optparse!(argv) # rubocop:disable Metrics/MethodLength
+    options = Options.new
+    option_parser = OptionParser.new do |parser|
+      parser.banner = "Usage: #{parser.program_name} [options]"
+      parser.on('-hHOST', '--host=HOST', 'Coursemology host to connect to') do |host|
+        options.host = host
+      end
+
+      parser.on('-tTOKEN', '--api-token=TOKEN') do |token|
+        options.api_token = token
+      end
+
+      parser.on('-uUSER', '--api-user-email=USER') do |user|
+        options.api_user_email = user
+      end
+
+      parser.on('-o', '--one-shot') do
+        options.one_shot = true
+      end
+    end
+
+    option_parser.parse!(argv)
+    options
+  end
+end

data/lib/coursemology/evaluator/client.rb

@@ -0,0 +1,74 @@
+class Coursemology::Evaluator::Client
+  def self.initialize(host, api_user_email, api_token)
+    Coursemology::Evaluator::Models::Base.base_url = host
+    Coursemology::Evaluator::Models::Base.api_user_email = api_user_email
+    Coursemology::Evaluator::Models::Base.api_token = api_token
+
+    Coursemology::Evaluator::Models::Base.initialize
+  end
+
+  # @param [Boolean] one_shot If the client should only fire one request.
+  def initialize(one_shot = false)
+    @terminate = one_shot
+  end
+
+  def run
+    Signal.trap('SIGTERM', method(:on_sig_term))
+
+    loop do
+      allocate_evaluations
+      break if @terminate
+
+      # :nocov:
+      # This sleep might not be triggered in the specs, because interruptions to the thread is
+      # nondeterministically run by the OS scheduler.
+      sleep(1.minute)
+      # :nocov:
+    end
+  end
+
+  private
+
+  # Requests evaluations from the server.
+  def allocate_evaluations
+    evaluations =
+      ActiveSupport::Notifications.instrument('allocate.client.evaluator.coursemology') do
+        languages = Coursemology::Polyglot::Language.concrete_languages.map(&:display_name)
+        Coursemology::Evaluator::Models::ProgrammingEvaluation.allocate(language: languages)
+      end
+
+    on_allocate(evaluations)
+  rescue Flexirest::HTTPUnauthorisedClientException => e
+    ActiveSupport::Notifications.publish('allocate_fail.client.evaluator.coursemology', e: e)
+  end
+
+  # The callback for handling an array of allocated evaluations.
+  #
+  # @param [Array<Coursemology::Evaluator::Models::ProgrammingEvaluation>] evaluations The
+  #   evaluations retrieved from the server.
+  def on_allocate(evaluations)
+    evaluations.each do |evaluation|
+      on_evaluation(evaluation)
+    end
+  end
+
+  # The callback for handling an evaluation.
+  #
+  # @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
+  #   retrieved from the server.
+  def on_evaluation(evaluation)
+    ActiveSupport::Notifications.instrument('evaluate.client.evaluator.coursemology',
+                                            evaluation: evaluation) do
+      evaluation.evaluate
+    end
+
+    ActiveSupport::Notifications.instrument('save.client.evaluator.coursemology') do
+      evaluation.save
+    end
+  end
+
+  # The callback for handling SIGTERM sent to the process.
+  def on_sig_term
+    @terminate = true
+  end
+end

data/lib/coursemology/evaluator/docker_container.rb

@@ -0,0 +1,58 @@
+class Coursemology::Evaluator::DockerContainer < Docker::Container
+  class << self
+    def create(image, argv: nil)
+      pull_image(image)
+
+      ActiveSupport::Notifications.instrument('create.docker.evaluator.coursemology',
+                                              image: image) do |payload|
+        options = { 'Image' => image }
+        options['Cmd'] = argv if argv && !argv.empty?
+
+        payload[:container] = super(options)
+      end
+    end
+
+    private
+
+    def pull_image(image)
+      ActiveSupport::Notifications.instrument('pull.docker.evaluator.coursemology',
+                                              image: image) do
+        Docker::Image.create('fromImage' => image)
+      end
+    end
+  end
+
+  # Waits for the container to exit the Running state.
+  #
+  # This will time out for long running operations, so keep retrying until we return.
+  #
+  # @param [Fixnum|nil] time The amount of time to wait.
+  # @return [Fixnum] The exit code of the container.
+  def wait(time = nil)
+    container_state = info
+    while container_state.fetch('State', {}).fetch('Running', true)
+      super
+      refresh!
+      container_state = info
+    end
+
+    container_state['State']['ExitCode']
+  rescue Docker::Error::TimeoutError
+    retry
+  end
+
+  # Gets the exit code of the container.
+  #
+  # @return [Fixnum] The exit code of the container, if +wait+ was called before.
+  # @return [nil] If the container is still running, or +wait+ was not called.
+  def exit_code
+    info.fetch('State', {})['ExitCode']
+  end
+
+  def delete
+    ActiveSupport::Notifications.instrument('destroy.docker.evaluator.coursemology',
+                                            container: id) do
+      super
+    end
+  end
+end

data/lib/coursemology/evaluator/logging.rb

@@ -0,0 +1,11 @@
+module Coursemology::Evaluator::Logging
+  extend ActiveSupport::Autoload
+
+  eager_autoload do
+    autoload :ClientLogSubscriber
+    autoload :DockerLogSubscriber
+  end
+end
+
+# Define +Rails+ to trick ActiveSupport into logging to our logger.
+Rails = Coursemology::Evaluator

data/lib/coursemology/evaluator/logging/client_log_subscriber.rb

@@ -0,0 +1,24 @@
+class Coursemology::Evaluator::Logging::ClientLogSubscriber < ActiveSupport::LogSubscriber
+  def publish(name, *args)
+    send(name.split('.').first, *args)
+  end
+
+  def allocate(event)
+    info color("Client: Allocate (#{event.duration.round(1)}ms)", MAGENTA)
+  end
+
+  def allocate_fail(e:)
+    error color("Client: Allocate failed: #{e.message}", RED)
+  end
+
+  def evaluate(event)
+    info "#{color("Client: Evaluate (#{event.duration.round(1)}ms)", CYAN)} "\
+      "#{event.payload[:evaluation].language.class.display_name}"
+  end
+
+  def save(event)
+    info color("Client: Save (#{event.duration.round(1)}ms)", GREEN)
+  end
+end
+
+Coursemology::Evaluator::Logging::ClientLogSubscriber.attach_to(:'client.evaluator.coursemology')

data/lib/coursemology/evaluator/logging/docker_log_subscriber.rb

@@ -0,0 +1,17 @@
+class Coursemology::Evaluator::Logging::DockerLogSubscriber < ActiveSupport::LogSubscriber
+  def pull(event)
+    info "#{color("Docker Pull (#{event.duration.round(1)}ms)", GREEN)} #{event.payload[:image]}"
+  end
+
+  def create(event)
+    info "#{color("Docker Create (#{event.duration.round(1)}ms)", MAGENTA)} "\
+      "#{event.payload[:image]} => #{event.payload[:container].id}"
+  end
+
+  def destroy(event)
+    info "#{color("Docker Destroy (#{event.duration.round(1)}ms)", CYAN)} "\
+      "#{event.payload[:container]}"
+  end
+end
+
+Coursemology::Evaluator::Logging::DockerLogSubscriber.attach_to(:'docker.evaluator.coursemology')

data/lib/coursemology/evaluator/models.rb

@@ -0,0 +1,6 @@
+module Coursemology::Evaluator::Models
+  extend ActiveSupport::Autoload
+
+  autoload :Base
+  autoload :ProgrammingEvaluation
+end

data/lib/coursemology/evaluator/models/base.rb

@@ -0,0 +1,49 @@
+class Coursemology::Evaluator::Models::Base < Flexirest::Base
+  class << self
+    attr_accessor :api_user_email
+    attr_accessor :api_token
+
+    def initialize
+      Flexirest::Base.perform_caching = false
+      default_config = Flexirest::Base.faraday_config
+      Flexirest::Base.faraday_config do |faraday|
+        # +follow_redirects+ must be added before declaring the adapter. See faraday_middleware#32,
+        # last comment.
+        faraday.response :follow_redirects
+
+        default_config.call(faraday)
+      end
+    end
+  end
+
+  verbose!
+  before_request :add_authentication
+
+  # Sets the key of the model. This is the key that all attributes are nested under, the same as
+  # the +require+ directive in the controller of the web application.
+  #
+  # @param [String] key The key to prefix all attributes with.
+  def self.model_key(key)
+    before_request do |name, param|
+      fix_put_parameters(key, name, param) if [:post, :patch, :put].include?(param.method[:method])
+    end
+  end
+  private_class_method :model_key
+
+  # Fixes the request parameters when executing a POST, PATCH or PUT.
+  #
+  # @param [String] key The key to prefix all attributes with.
+  # @param [Request] param The request parameter to prepend the key with.
+  def self.fix_put_parameters(key, _, param)
+    param.post_params = { key => param.post_params } unless param.post_params.empty?
+  end
+  private_class_method :fix_put_parameters
+
+  private
+
+  # Adds the authentication email and token to the request.
+  def add_authentication(_, request)
+    request.headers['X-User-Email'] = self.class.api_user_email
+    request.headers['X-User-Token'] = self.class.api_token
+  end
+end

data/lib/coursemology/evaluator/models/programming_evaluation.rb

@@ -0,0 +1,66 @@
+class Coursemology::Evaluator::Models::ProgrammingEvaluation < Coursemology::Evaluator::Models::Base
+  extend ActiveSupport::Autoload
+  autoload :Package
+
+  request_body_type :json
+  model_key :programming_evaluation
+
+  get :find, 'courses/assessment/programming_evaluations/:id'.freeze
+  post :allocate, 'courses/assessment/programming_evaluations/allocate'.freeze
+  put :save, 'courses/assessment/programming_evaluations/:id/result'.freeze
+
+  # Gets the language for the programming evaluation.
+  #
+  # @return [nil] If the language does not exist.
+  # @return [Coursemology::Polyglot::Language] The language that the evaluation uses.
+  def language
+    Coursemology::Polyglot::Language.find_by(type: super)
+  end
+
+  # Sets the language for the programming evaluation.
+  #
+  # @param [String|nil|Coursemology::Polyglot::Language] language The language to set. If this is
+  #   a string, it is assumed to be the class name of the language.
+  def language=(language)
+    return super(language) if language.nil? || language.is_a?(String)
+
+    fail ArgumentError unless language.is_a?(Coursemology::Polyglot::Language)
+    super(language.class.name)
+  end
+
+  # Obtains the package.
+  #
+  # @return [Coursemology::Evaluator::Models::ProgrammingEvaluation::Package]
+  def package
+    @package ||= begin
+      body = plain_request('courses/assessment/programming_evaluations/:id/package', id: id)
+      Package.new(Coursemology::Evaluator::StringIO.new(body))
+    end
+  end
+
+  # Evaluates the package, and stores the result in this record.
+  #
+  # Call {Coursemology::Evaluator::Models::ProgrammingEvaluation#save} to save the record to the
+  # server.
+  def evaluate
+    result = Coursemology::Evaluator::Services::EvaluateProgrammingPackageService.
+             execute(self)
+    self.stdout = result.stdout
+    self.stderr = result.stderr
+    self.test_report = result.test_report
+    self.exit_code = result.exit_code
+  end
+
+  private
+
+  # Performs a plain request.
+  #
+  # @param [String] url The URL to request.
+  # @param [Hash] params The parameter to be part of the request.
+  # @return [String] The response body.
+  def plain_request(url, params = {})
+    request = Flexirest::Request.new({ url: url, method: :get, options: { plain: true } },
+                                     self.class)
+    request.call(params)
+  end
+end