coursemology-evaluator 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.env +3 -3
- data/.gitignore +23 -23
- data/.hound.yml +8 -8
- data/.idea/Coursemology Evaluator.iml +8 -8
- data/.rspec +2 -2
- data/.rubocop.unhound.yml +109 -109
- data/.rubocop.yml +46 -46
- data/.travis.yml +17 -17
- data/Gemfile +4 -4
- data/Procfile +1 -1
- data/README.md +52 -52
- data/Rakefile +6 -6
- data/bin/evaluator +5 -5
- data/coursemology-evaluator.gemspec +38 -38
- data/lib/coursemology/evaluator.rb +41 -41
- data/lib/coursemology/evaluator/cli.rb +74 -74
- data/lib/coursemology/evaluator/client.rb +84 -84
- data/lib/coursemology/evaluator/docker_container.rb +79 -79
- data/lib/coursemology/evaluator/logging.rb +12 -12
- data/lib/coursemology/evaluator/logging/client_log_subscriber.rb +25 -25
- data/lib/coursemology/evaluator/logging/docker_log_subscriber.rb +21 -21
- data/lib/coursemology/evaluator/models.rb +7 -7
- data/lib/coursemology/evaluator/models/base.rb +50 -50
- data/lib/coursemology/evaluator/models/programming_evaluation.rb +55 -55
- data/lib/coursemology/evaluator/models/programming_evaluation/package.rb +12 -12
- data/lib/coursemology/evaluator/services.rb +6 -6
- data/lib/coursemology/evaluator/services/evaluate_programming_package_service.rb +151 -151
- data/lib/coursemology/evaluator/string_io.rb +14 -14
- data/lib/coursemology/evaluator/utils.rb +42 -42
- data/lib/coursemology/evaluator/version.rb +5 -5
- data/lib/coursemology/polyglot/extensions.rb +3 -3
- data/lib/coursemology/polyglot/extensions/language.rb +24 -24
- metadata +3 -3
@@ -1,84 +1,84 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::Client
|
3
|
-
def self.initialize(host, api_user_email, api_token)
|
4
|
-
Coursemology::Evaluator::Models::Base.base_url = host
|
5
|
-
Coursemology::Evaluator::Models::Base.api_user_email = api_user_email
|
6
|
-
Coursemology::Evaluator::Models::Base.api_token = api_token
|
7
|
-
|
8
|
-
Coursemology::Evaluator::Models::Base.initialize
|
9
|
-
end
|
10
|
-
|
11
|
-
# @param [Boolean] one_shot If the client should only fire one request.
|
12
|
-
def initialize(one_shot = false)
|
13
|
-
@terminate = one_shot
|
14
|
-
end
|
15
|
-
|
16
|
-
def run
|
17
|
-
Signal.trap('SIGTERM', method(:on_sig_term))
|
18
|
-
loop(&method(:client_loop))
|
19
|
-
end
|
20
|
-
|
21
|
-
private
|
22
|
-
|
23
|
-
# Performs one iteration of the client loop.
|
24
|
-
def client_loop
|
25
|
-
evaluations = allocate_evaluations
|
26
|
-
if evaluations && !evaluations.empty?
|
27
|
-
on_allocate(evaluations)
|
28
|
-
else
|
29
|
-
raise StopIteration if @terminate
|
30
|
-
|
31
|
-
# :nocov:
|
32
|
-
# This sleep might not be triggered in the specs, because interruptions to the thread is
|
33
|
-
# nondeterministically run by the OS scheduler.
|
34
|
-
sleep(Coursemology::Evaluator.config.poll_interval)
|
35
|
-
# :nocov:
|
36
|
-
end
|
37
|
-
|
38
|
-
raise StopIteration if @terminate
|
39
|
-
end
|
40
|
-
|
41
|
-
# Requests evaluations from the server.
|
42
|
-
#
|
43
|
-
# @return [Array<Coursemology::Evaluator::Models::ProgrammingEvaluation>] The evaluations
|
44
|
-
# retrieved from the server.
|
45
|
-
def allocate_evaluations
|
46
|
-
ActiveSupport::Notifications.instrument('allocate.client.evaluator.coursemology') do
|
47
|
-
languages = Coursemology::Polyglot::Language.concrete_languages.map(&:display_name)
|
48
|
-
Coursemology::Evaluator::Models::ProgrammingEvaluation.allocate(language: languages)
|
49
|
-
end
|
50
|
-
rescue Flexirest::HTTPUnauthorisedClientException => e
|
51
|
-
ActiveSupport::Notifications.publish('allocate_fail.client.evaluator.coursemology', e: e)
|
52
|
-
nil
|
53
|
-
end
|
54
|
-
|
55
|
-
# The callback for handling an array of allocated evaluations.
|
56
|
-
#
|
57
|
-
# @param [Array<Coursemology::Evaluator::Models::ProgrammingEvaluation>] evaluations The
|
58
|
-
# evaluations retrieved from the server.
|
59
|
-
def on_allocate(evaluations)
|
60
|
-
evaluations.each do |evaluation|
|
61
|
-
on_evaluation(evaluation)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
|
65
|
-
# The callback for handling an evaluation.
|
66
|
-
#
|
67
|
-
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
|
68
|
-
# retrieved from the server.
|
69
|
-
def on_evaluation(evaluation)
|
70
|
-
ActiveSupport::Notifications.instrument('evaluate.client.evaluator.coursemology',
|
71
|
-
evaluation: evaluation) do
|
72
|
-
evaluation.evaluate
|
73
|
-
end
|
74
|
-
|
75
|
-
ActiveSupport::Notifications.instrument('save.client.evaluator.coursemology') do
|
76
|
-
evaluation.save
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
# The callback for handling SIGTERM sent to the process.
|
81
|
-
def on_sig_term
|
82
|
-
@terminate = true
|
83
|
-
end
|
84
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::Client
|
3
|
+
def self.initialize(host, api_user_email, api_token)
|
4
|
+
Coursemology::Evaluator::Models::Base.base_url = host
|
5
|
+
Coursemology::Evaluator::Models::Base.api_user_email = api_user_email
|
6
|
+
Coursemology::Evaluator::Models::Base.api_token = api_token
|
7
|
+
|
8
|
+
Coursemology::Evaluator::Models::Base.initialize
|
9
|
+
end
|
10
|
+
|
11
|
+
# @param [Boolean] one_shot If the client should only fire one request.
|
12
|
+
def initialize(one_shot = false)
|
13
|
+
@terminate = one_shot
|
14
|
+
end
|
15
|
+
|
16
|
+
def run
|
17
|
+
Signal.trap('SIGTERM', method(:on_sig_term))
|
18
|
+
loop(&method(:client_loop))
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
|
23
|
+
# Performs one iteration of the client loop.
|
24
|
+
def client_loop
|
25
|
+
evaluations = allocate_evaluations
|
26
|
+
if evaluations && !evaluations.empty?
|
27
|
+
on_allocate(evaluations)
|
28
|
+
else
|
29
|
+
raise StopIteration if @terminate
|
30
|
+
|
31
|
+
# :nocov:
|
32
|
+
# This sleep might not be triggered in the specs, because interruptions to the thread is
|
33
|
+
# nondeterministically run by the OS scheduler.
|
34
|
+
sleep(Coursemology::Evaluator.config.poll_interval)
|
35
|
+
# :nocov:
|
36
|
+
end
|
37
|
+
|
38
|
+
raise StopIteration if @terminate
|
39
|
+
end
|
40
|
+
|
41
|
+
# Requests evaluations from the server.
|
42
|
+
#
|
43
|
+
# @return [Array<Coursemology::Evaluator::Models::ProgrammingEvaluation>] The evaluations
|
44
|
+
# retrieved from the server.
|
45
|
+
def allocate_evaluations
|
46
|
+
ActiveSupport::Notifications.instrument('allocate.client.evaluator.coursemology') do
|
47
|
+
languages = Coursemology::Polyglot::Language.concrete_languages.map(&:display_name)
|
48
|
+
Coursemology::Evaluator::Models::ProgrammingEvaluation.allocate(language: languages)
|
49
|
+
end
|
50
|
+
rescue Flexirest::HTTPUnauthorisedClientException => e
|
51
|
+
ActiveSupport::Notifications.publish('allocate_fail.client.evaluator.coursemology', e: e)
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
|
55
|
+
# The callback for handling an array of allocated evaluations.
|
56
|
+
#
|
57
|
+
# @param [Array<Coursemology::Evaluator::Models::ProgrammingEvaluation>] evaluations The
|
58
|
+
# evaluations retrieved from the server.
|
59
|
+
def on_allocate(evaluations)
|
60
|
+
evaluations.each do |evaluation|
|
61
|
+
on_evaluation(evaluation)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
# The callback for handling an evaluation.
|
66
|
+
#
|
67
|
+
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
|
68
|
+
# retrieved from the server.
|
69
|
+
def on_evaluation(evaluation)
|
70
|
+
ActiveSupport::Notifications.instrument('evaluate.client.evaluator.coursemology',
|
71
|
+
evaluation: evaluation) do
|
72
|
+
evaluation.evaluate
|
73
|
+
end
|
74
|
+
|
75
|
+
ActiveSupport::Notifications.instrument('save.client.evaluator.coursemology') do
|
76
|
+
evaluation.save
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# The callback for handling SIGTERM sent to the process.
|
81
|
+
def on_sig_term
|
82
|
+
@terminate = true
|
83
|
+
end
|
84
|
+
end
|
@@ -1,79 +1,79 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::DockerContainer < Docker::Container
|
3
|
-
class << self
|
4
|
-
def create(image, argv: nil)
|
5
|
-
pull_image(image)
|
6
|
-
|
7
|
-
ActiveSupport::Notifications.instrument('create.docker.evaluator.coursemology',
|
8
|
-
image: image) do |payload|
|
9
|
-
options = { 'Image' => image }
|
10
|
-
options['Cmd'] = argv if argv && !argv.empty?
|
11
|
-
|
12
|
-
payload[:container] = super(options)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
private
|
17
|
-
|
18
|
-
# Pulls the given image from Docker Hub.
|
19
|
-
#
|
20
|
-
# This caches images for the specified time, because the overhead for querying
|
21
|
-
# for images is quite high.
|
22
|
-
#
|
23
|
-
# @param [String] image The image to pull.
|
24
|
-
def pull_image(image)
|
25
|
-
ActiveSupport::Notifications.instrument('pull.docker.evaluator.coursemology',
|
26
|
-
image: image) do |payload|
|
27
|
-
cached([:image, image], expires_in: Coursemology::Evaluator.config.image_lifetime) do
|
28
|
-
Docker::Image.create('fromImage' => image)
|
29
|
-
payload[:cached] = false
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
# Cache the result of the given block using the key given.
|
35
|
-
#
|
36
|
-
# @param [Array, String, Symbol] key The key to use. This will be expanded with
|
37
|
-
# +ActiveSupport::Cache.expand_cache_key+.
|
38
|
-
# @param [Hash] options The options to use. These are the same as
|
39
|
-
# +ActiveSupport::Cache::Store#fetch+.
|
40
|
-
def cached(key, options = {}, &proc)
|
41
|
-
key = ActiveSupport::Cache.expand_cache_key(key, name.underscore)
|
42
|
-
Coursemology::Evaluator.cache.fetch(key, options, &proc)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
# Waits for the container to exit the Running state.
|
47
|
-
#
|
48
|
-
# This will time out for long running operations, so keep retrying until we return.
|
49
|
-
#
|
50
|
-
# @param [Fixnum|nil] time The amount of time to wait.
|
51
|
-
# @return [Fixnum] The exit code of the container.
|
52
|
-
def wait(time = nil)
|
53
|
-
container_state = info
|
54
|
-
while container_state.fetch('State', {}).fetch('Running', true)
|
55
|
-
super
|
56
|
-
refresh!
|
57
|
-
container_state = info
|
58
|
-
end
|
59
|
-
|
60
|
-
container_state['State']['ExitCode']
|
61
|
-
rescue Docker::Error::TimeoutError
|
62
|
-
retry
|
63
|
-
end
|
64
|
-
|
65
|
-
# Gets the exit code of the container.
|
66
|
-
#
|
67
|
-
# @return [Fixnum] The exit code of the container, if +wait+ was called before.
|
68
|
-
# @return [nil] If the container is still running, or +wait+ was not called.
|
69
|
-
def exit_code
|
70
|
-
info.fetch('State', {})['ExitCode']
|
71
|
-
end
|
72
|
-
|
73
|
-
def delete
|
74
|
-
ActiveSupport::Notifications.instrument('destroy.docker.evaluator.coursemology',
|
75
|
-
container: id) do
|
76
|
-
super
|
77
|
-
end
|
78
|
-
end
|
79
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::DockerContainer < Docker::Container
|
3
|
+
class << self
|
4
|
+
def create(image, argv: nil)
|
5
|
+
pull_image(image)
|
6
|
+
|
7
|
+
ActiveSupport::Notifications.instrument('create.docker.evaluator.coursemology',
|
8
|
+
image: image) do |payload|
|
9
|
+
options = { 'Image' => image }
|
10
|
+
options['Cmd'] = argv if argv && !argv.empty?
|
11
|
+
|
12
|
+
payload[:container] = super(options)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
# Pulls the given image from Docker Hub.
|
19
|
+
#
|
20
|
+
# This caches images for the specified time, because the overhead for querying
|
21
|
+
# for images is quite high.
|
22
|
+
#
|
23
|
+
# @param [String] image The image to pull.
|
24
|
+
def pull_image(image)
|
25
|
+
ActiveSupport::Notifications.instrument('pull.docker.evaluator.coursemology',
|
26
|
+
image: image) do |payload|
|
27
|
+
cached([:image, image], expires_in: Coursemology::Evaluator.config.image_lifetime) do
|
28
|
+
Docker::Image.create('fromImage' => image)
|
29
|
+
payload[:cached] = false
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Cache the result of the given block using the key given.
|
35
|
+
#
|
36
|
+
# @param [Array, String, Symbol] key The key to use. This will be expanded with
|
37
|
+
# +ActiveSupport::Cache.expand_cache_key+.
|
38
|
+
# @param [Hash] options The options to use. These are the same as
|
39
|
+
# +ActiveSupport::Cache::Store#fetch+.
|
40
|
+
def cached(key, options = {}, &proc)
|
41
|
+
key = ActiveSupport::Cache.expand_cache_key(key, name.underscore)
|
42
|
+
Coursemology::Evaluator.cache.fetch(key, options, &proc)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# Waits for the container to exit the Running state.
|
47
|
+
#
|
48
|
+
# This will time out for long running operations, so keep retrying until we return.
|
49
|
+
#
|
50
|
+
# @param [Fixnum|nil] time The amount of time to wait.
|
51
|
+
# @return [Fixnum] The exit code of the container.
|
52
|
+
def wait(time = nil)
|
53
|
+
container_state = info
|
54
|
+
while container_state.fetch('State', {}).fetch('Running', true)
|
55
|
+
super
|
56
|
+
refresh!
|
57
|
+
container_state = info
|
58
|
+
end
|
59
|
+
|
60
|
+
container_state['State']['ExitCode']
|
61
|
+
rescue Docker::Error::TimeoutError
|
62
|
+
retry
|
63
|
+
end
|
64
|
+
|
65
|
+
# Gets the exit code of the container.
|
66
|
+
#
|
67
|
+
# @return [Fixnum] The exit code of the container, if +wait+ was called before.
|
68
|
+
# @return [nil] If the container is still running, or +wait+ was not called.
|
69
|
+
def exit_code
|
70
|
+
info.fetch('State', {})['ExitCode']
|
71
|
+
end
|
72
|
+
|
73
|
+
def delete
|
74
|
+
ActiveSupport::Notifications.instrument('destroy.docker.evaluator.coursemology',
|
75
|
+
container: id) do
|
76
|
+
super
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -1,12 +1,12 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Coursemology::Evaluator::Logging
|
3
|
-
extend ActiveSupport::Autoload
|
4
|
-
|
5
|
-
eager_autoload do
|
6
|
-
autoload :ClientLogSubscriber
|
7
|
-
autoload :DockerLogSubscriber
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
# Define +Rails+ to trick ActiveSupport into logging to our logger.
|
12
|
-
Rails = Coursemology::Evaluator
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Coursemology::Evaluator::Logging
|
3
|
+
extend ActiveSupport::Autoload
|
4
|
+
|
5
|
+
eager_autoload do
|
6
|
+
autoload :ClientLogSubscriber
|
7
|
+
autoload :DockerLogSubscriber
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Define +Rails+ to trick ActiveSupport into logging to our logger.
|
12
|
+
Rails = Coursemology::Evaluator
|
@@ -1,25 +1,25 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::Logging::ClientLogSubscriber < ActiveSupport::LogSubscriber
|
3
|
-
def publish(name, *args)
|
4
|
-
send(name.split('.').first, *args)
|
5
|
-
end
|
6
|
-
|
7
|
-
def allocate(event)
|
8
|
-
info color("Client: Allocate (#{event.duration.round(1)}ms)", MAGENTA)
|
9
|
-
end
|
10
|
-
|
11
|
-
def allocate_fail(e:)
|
12
|
-
error color("Client: Allocate failed: #{e.message}", RED)
|
13
|
-
end
|
14
|
-
|
15
|
-
def evaluate(event)
|
16
|
-
info "#{color("Client: Evaluate (#{event.duration.round(1)}ms)", CYAN)} "\
|
17
|
-
"#{event.payload[:evaluation].language.class.display_name}"
|
18
|
-
end
|
19
|
-
|
20
|
-
def save(event)
|
21
|
-
info color("Client: Save (#{event.duration.round(1)}ms)", GREEN)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
Coursemology::Evaluator::Logging::ClientLogSubscriber.attach_to(:'client.evaluator.coursemology')
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::Logging::ClientLogSubscriber < ActiveSupport::LogSubscriber
|
3
|
+
def publish(name, *args)
|
4
|
+
send(name.split('.').first, *args)
|
5
|
+
end
|
6
|
+
|
7
|
+
def allocate(event)
|
8
|
+
info color("Client: Allocate (#{event.duration.round(1)}ms)", MAGENTA)
|
9
|
+
end
|
10
|
+
|
11
|
+
def allocate_fail(e:)
|
12
|
+
error color("Client: Allocate failed: #{e.message}", RED)
|
13
|
+
end
|
14
|
+
|
15
|
+
def evaluate(event)
|
16
|
+
info "#{color("Client: Evaluate (#{event.duration.round(1)}ms)", CYAN)} "\
|
17
|
+
"#{event.payload[:evaluation].language.class.display_name}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def save(event)
|
21
|
+
info color("Client: Save (#{event.duration.round(1)}ms)", GREEN)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
Coursemology::Evaluator::Logging::ClientLogSubscriber.attach_to(:'client.evaluator.coursemology')
|
@@ -1,21 +1,21 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::Logging::DockerLogSubscriber < ActiveSupport::LogSubscriber
|
3
|
-
def pull(event)
|
4
|
-
cached = event.payload[:cached].nil? || event.payload[:cached] ? 'Cached ' : ''
|
5
|
-
header_colour = cached ? GREEN : YELLOW
|
6
|
-
info "#{color("#{cached}Docker Pull (#{event.duration.round(1)}ms)", header_colour)} "\
|
7
|
-
"#{event.payload[:image]}"
|
8
|
-
end
|
9
|
-
|
10
|
-
def create(event)
|
11
|
-
info "#{color("Docker Create (#{event.duration.round(1)}ms)", MAGENTA)} "\
|
12
|
-
"#{event.payload[:image]} => #{event.payload[:container].id}"
|
13
|
-
end
|
14
|
-
|
15
|
-
def destroy(event)
|
16
|
-
info "#{color("Docker Destroy (#{event.duration.round(1)}ms)", CYAN)} "\
|
17
|
-
"#{event.payload[:container]}"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
Coursemology::Evaluator::Logging::DockerLogSubscriber.attach_to(:'docker.evaluator.coursemology')
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::Logging::DockerLogSubscriber < ActiveSupport::LogSubscriber
|
3
|
+
def pull(event)
|
4
|
+
cached = event.payload[:cached].nil? || event.payload[:cached] ? 'Cached ' : ''
|
5
|
+
header_colour = cached ? GREEN : YELLOW
|
6
|
+
info "#{color("#{cached}Docker Pull (#{event.duration.round(1)}ms)", header_colour)} "\
|
7
|
+
"#{event.payload[:image]}"
|
8
|
+
end
|
9
|
+
|
10
|
+
def create(event)
|
11
|
+
info "#{color("Docker Create (#{event.duration.round(1)}ms)", MAGENTA)} "\
|
12
|
+
"#{event.payload[:image]} => #{event.payload[:container].id}"
|
13
|
+
end
|
14
|
+
|
15
|
+
def destroy(event)
|
16
|
+
info "#{color("Docker Destroy (#{event.duration.round(1)}ms)", CYAN)} "\
|
17
|
+
"#{event.payload[:container]}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
Coursemology::Evaluator::Logging::DockerLogSubscriber.attach_to(:'docker.evaluator.coursemology')
|
@@ -1,7 +1,7 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Coursemology::Evaluator::Models
|
3
|
-
extend ActiveSupport::Autoload
|
4
|
-
|
5
|
-
autoload :Base
|
6
|
-
autoload :ProgrammingEvaluation
|
7
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Coursemology::Evaluator::Models
|
3
|
+
extend ActiveSupport::Autoload
|
4
|
+
|
5
|
+
autoload :Base
|
6
|
+
autoload :ProgrammingEvaluation
|
7
|
+
end
|