coursemology-evaluator 0.1.1 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.env +3 -3
- data/.gitignore +23 -22
- data/.hound.yml +8 -8
- data/.idea/Coursemology Evaluator.iml +8 -8
- data/.rspec +2 -2
- data/.rubocop.unhound.yml +109 -109
- data/.rubocop.yml +46 -46
- data/.travis.yml +17 -17
- data/Gemfile +4 -4
- data/Procfile +1 -1
- data/README.md +29 -29
- data/Rakefile +6 -6
- data/bin/evaluator +5 -5
- data/coursemology-evaluator.gemspec +37 -37
- data/lib/coursemology/evaluator.rb +36 -36
- data/lib/coursemology/evaluator/cli.rb +52 -52
- data/lib/coursemology/evaluator/client.rb +81 -75
- data/lib/coursemology/evaluator/docker_container.rb +59 -59
- data/lib/coursemology/evaluator/logging.rb +12 -12
- data/lib/coursemology/evaluator/logging/client_log_subscriber.rb +25 -25
- data/lib/coursemology/evaluator/logging/docker_log_subscriber.rb +18 -18
- data/lib/coursemology/evaluator/models.rb +7 -7
- data/lib/coursemology/evaluator/models/base.rb +50 -50
- data/lib/coursemology/evaluator/models/programming_evaluation.rb +55 -67
- data/lib/coursemology/evaluator/models/programming_evaluation/package.rb +12 -12
- data/lib/coursemology/evaluator/services.rb +6 -6
- data/lib/coursemology/evaluator/services/evaluate_programming_package_service.rb +151 -151
- data/lib/coursemology/evaluator/string_io.rb +14 -14
- data/lib/coursemology/evaluator/utils.rb +42 -42
- data/lib/coursemology/evaluator/version.rb +5 -5
- data/lib/coursemology/polyglot/extensions.rb +3 -3
- data/lib/coursemology/polyglot/extensions/language.rb +24 -24
- metadata +15 -4
- data/Gemfile.lock +0 -114
@@ -1,12 +1,12 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::Models::ProgrammingEvaluation::Package
|
3
|
-
# The stream comprising the package.
|
4
|
-
attr_reader :stream
|
5
|
-
|
6
|
-
# Constructs a new Package.
|
7
|
-
#
|
8
|
-
# @param [IO] stream The stream comprising the package.
|
9
|
-
def initialize(stream)
|
10
|
-
@stream = stream
|
11
|
-
end
|
12
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::Models::ProgrammingEvaluation::Package
|
3
|
+
# The stream comprising the package.
|
4
|
+
attr_reader :stream
|
5
|
+
|
6
|
+
# Constructs a new Package.
|
7
|
+
#
|
8
|
+
# @param [IO] stream The stream comprising the package.
|
9
|
+
def initialize(stream)
|
10
|
+
@stream = stream
|
11
|
+
end
|
12
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Coursemology::Evaluator::Services
|
3
|
-
extend ActiveSupport::Autoload
|
4
|
-
|
5
|
-
autoload :EvaluateProgrammingPackageService
|
6
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Coursemology::Evaluator::Services
|
3
|
+
extend ActiveSupport::Autoload
|
4
|
+
|
5
|
+
autoload :EvaluateProgrammingPackageService
|
6
|
+
end
|
@@ -1,151 +1,151 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
class Coursemology::Evaluator::Services::EvaluateProgrammingPackageService
|
3
|
-
Result = Struct.new(:stdout, :stderr, :test_report, :exit_code)
|
4
|
-
|
5
|
-
# The path to the Coursemology user home directory.
|
6
|
-
HOME_PATH = '/home/coursemology'.freeze
|
7
|
-
|
8
|
-
# The path to where the package will be extracted.
|
9
|
-
PACKAGE_PATH = File.join(HOME_PATH, 'package')
|
10
|
-
|
11
|
-
# The path to where the test report will be at.
|
12
|
-
REPORT_PATH = File.join(PACKAGE_PATH, 'report.xml')
|
13
|
-
|
14
|
-
# The ratio to multiply the memory limits from our evaluation to the container by.
|
15
|
-
MEMORY_LIMIT_RATIO = 1.megabyte / 1.kilobyte
|
16
|
-
|
17
|
-
# Executes the given package in a container.
|
18
|
-
#
|
19
|
-
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
|
20
|
-
# from the server.
|
21
|
-
# @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result] The
|
22
|
-
# result of the evaluation.
|
23
|
-
def self.execute(evaluation)
|
24
|
-
new(evaluation).send(:execute)
|
25
|
-
end
|
26
|
-
|
27
|
-
# Creates a new service object.
|
28
|
-
def initialize(evaluation)
|
29
|
-
@evaluation = evaluation
|
30
|
-
@package = evaluation.package
|
31
|
-
end
|
32
|
-
|
33
|
-
private
|
34
|
-
|
35
|
-
# Evaluates the package.
|
36
|
-
#
|
37
|
-
# @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result]
|
38
|
-
def execute
|
39
|
-
container = create_container(@evaluation.language.class.docker_image)
|
40
|
-
copy_package(container)
|
41
|
-
execute_package(container)
|
42
|
-
|
43
|
-
extract_result(container)
|
44
|
-
ensure
|
45
|
-
destroy_container(container) if container
|
46
|
-
end
|
47
|
-
|
48
|
-
def create_container(image)
|
49
|
-
image_identifier = "coursemology/evaluator-image-#{image}"
|
50
|
-
Coursemology::Evaluator::DockerContainer.create(image_identifier, argv: container_arguments)
|
51
|
-
end
|
52
|
-
|
53
|
-
def container_arguments
|
54
|
-
result = []
|
55
|
-
result.push("-c#{@evaluation.time_limit}") if @evaluation.time_limit
|
56
|
-
result.push("-m#{@evaluation.memory_limit * MEMORY_LIMIT_RATIO}") if @evaluation.memory_limit
|
57
|
-
|
58
|
-
result
|
59
|
-
end
|
60
|
-
|
61
|
-
# Copies the contents of the package to the container.
|
62
|
-
#
|
63
|
-
# @param [Docker::Container] container The container to copy the package into.
|
64
|
-
def copy_package(container)
|
65
|
-
tar = tar_package(@package)
|
66
|
-
container.archive_in_stream(HOME_PATH) do
|
67
|
-
tar.read(Excon.defaults[:chunk_size]).to_s
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
# Converts the zip package into a tar package for the container.
|
72
|
-
#
|
73
|
-
# This also adds an additional +package+ directory to the start of the path, following tar
|
74
|
-
# convention.
|
75
|
-
#
|
76
|
-
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation::Package] package The package
|
77
|
-
# to convert to a tar.
|
78
|
-
# @return [IO] A stream containing the tar.
|
79
|
-
def tar_package(package)
|
80
|
-
tar_file_stream = StringIO.new
|
81
|
-
tar_file = Gem::Package::TarWriter.new(tar_file_stream)
|
82
|
-
Zip::File.open_buffer(package.stream) do |zip_file|
|
83
|
-
copy_archive(zip_file, tar_file, File.basename(PACKAGE_PATH))
|
84
|
-
tar_file.close
|
85
|
-
end
|
86
|
-
|
87
|
-
tar_file_stream.seek(0)
|
88
|
-
tar_file_stream
|
89
|
-
end
|
90
|
-
|
91
|
-
# Copies every entry from the zip archive to the tar archive, adding the optional prefix to the
|
92
|
-
# start of each file name.
|
93
|
-
#
|
94
|
-
# @param [Zip::File] zip_file The zip file to read from.
|
95
|
-
# @param [Gem::Package::TarWriter] tar_file The tar file to write to.
|
96
|
-
# @param [String] prefix The prefix to add to every file name in the tar.
|
97
|
-
def copy_archive(zip_file, tar_file, prefix = nil)
|
98
|
-
zip_file.each do |entry|
|
99
|
-
next unless entry.file?
|
100
|
-
|
101
|
-
zip_entry_stream = entry.get_input_stream
|
102
|
-
new_entry_name = prefix ? File.join(prefix, entry.name) : entry.name
|
103
|
-
tar_file.add_file(new_entry_name, 0664) do |tar_entry_stream|
|
104
|
-
IO.copy_stream(zip_entry_stream, tar_entry_stream)
|
105
|
-
end
|
106
|
-
|
107
|
-
zip_entry_stream.close
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
def execute_package(container)
|
112
|
-
container.start!
|
113
|
-
container.wait
|
114
|
-
end
|
115
|
-
|
116
|
-
def extract_result(container)
|
117
|
-
logs = container.logs(stdout: true, stderr: true)
|
118
|
-
|
119
|
-
_, stdout, stderr = Coursemology::Evaluator::Utils.parse_docker_stream(logs)
|
120
|
-
Result.new(stdout, stderr, extract_test_report(container), container.exit_code)
|
121
|
-
end
|
122
|
-
|
123
|
-
def extract_test_report(container)
|
124
|
-
stream = extract_test_report_archive(container)
|
125
|
-
|
126
|
-
tar_file = Gem::Package::TarReader.new(stream)
|
127
|
-
tar_file.each do |file|
|
128
|
-
return file.read
|
129
|
-
end
|
130
|
-
rescue Docker::Error::NotFoundError
|
131
|
-
return nil
|
132
|
-
end
|
133
|
-
|
134
|
-
# Extracts the test report from the container.
|
135
|
-
#
|
136
|
-
# @return [StringIO] The stream containing the archive, the pointer is reset to the start of the
|
137
|
-
# stream.
|
138
|
-
def extract_test_report_archive(container)
|
139
|
-
stream = StringIO.new
|
140
|
-
container.archive_out(REPORT_PATH) do |bytes|
|
141
|
-
stream.write(bytes)
|
142
|
-
end
|
143
|
-
|
144
|
-
stream.seek(0)
|
145
|
-
stream
|
146
|
-
end
|
147
|
-
|
148
|
-
def destroy_container(container)
|
149
|
-
container.delete
|
150
|
-
end
|
151
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
class Coursemology::Evaluator::Services::EvaluateProgrammingPackageService
|
3
|
+
Result = Struct.new(:stdout, :stderr, :test_report, :exit_code)
|
4
|
+
|
5
|
+
# The path to the Coursemology user home directory.
|
6
|
+
HOME_PATH = '/home/coursemology'.freeze
|
7
|
+
|
8
|
+
# The path to where the package will be extracted.
|
9
|
+
PACKAGE_PATH = File.join(HOME_PATH, 'package')
|
10
|
+
|
11
|
+
# The path to where the test report will be at.
|
12
|
+
REPORT_PATH = File.join(PACKAGE_PATH, 'report.xml')
|
13
|
+
|
14
|
+
# The ratio to multiply the memory limits from our evaluation to the container by.
|
15
|
+
MEMORY_LIMIT_RATIO = 1.megabyte / 1.kilobyte
|
16
|
+
|
17
|
+
# Executes the given package in a container.
|
18
|
+
#
|
19
|
+
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
|
20
|
+
# from the server.
|
21
|
+
# @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result] The
|
22
|
+
# result of the evaluation.
|
23
|
+
def self.execute(evaluation)
|
24
|
+
new(evaluation).send(:execute)
|
25
|
+
end
|
26
|
+
|
27
|
+
# Creates a new service object.
|
28
|
+
def initialize(evaluation)
|
29
|
+
@evaluation = evaluation
|
30
|
+
@package = evaluation.package
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
# Evaluates the package.
|
36
|
+
#
|
37
|
+
# @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result]
|
38
|
+
def execute
|
39
|
+
container = create_container(@evaluation.language.class.docker_image)
|
40
|
+
copy_package(container)
|
41
|
+
execute_package(container)
|
42
|
+
|
43
|
+
extract_result(container)
|
44
|
+
ensure
|
45
|
+
destroy_container(container) if container
|
46
|
+
end
|
47
|
+
|
48
|
+
def create_container(image)
|
49
|
+
image_identifier = "coursemology/evaluator-image-#{image}"
|
50
|
+
Coursemology::Evaluator::DockerContainer.create(image_identifier, argv: container_arguments)
|
51
|
+
end
|
52
|
+
|
53
|
+
def container_arguments
|
54
|
+
result = []
|
55
|
+
result.push("-c#{@evaluation.time_limit}") if @evaluation.time_limit
|
56
|
+
result.push("-m#{@evaluation.memory_limit * MEMORY_LIMIT_RATIO}") if @evaluation.memory_limit
|
57
|
+
|
58
|
+
result
|
59
|
+
end
|
60
|
+
|
61
|
+
# Copies the contents of the package to the container.
|
62
|
+
#
|
63
|
+
# @param [Docker::Container] container The container to copy the package into.
|
64
|
+
def copy_package(container)
|
65
|
+
tar = tar_package(@package)
|
66
|
+
container.archive_in_stream(HOME_PATH) do
|
67
|
+
tar.read(Excon.defaults[:chunk_size]).to_s
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Converts the zip package into a tar package for the container.
|
72
|
+
#
|
73
|
+
# This also adds an additional +package+ directory to the start of the path, following tar
|
74
|
+
# convention.
|
75
|
+
#
|
76
|
+
# @param [Coursemology::Evaluator::Models::ProgrammingEvaluation::Package] package The package
|
77
|
+
# to convert to a tar.
|
78
|
+
# @return [IO] A stream containing the tar.
|
79
|
+
def tar_package(package)
|
80
|
+
tar_file_stream = StringIO.new
|
81
|
+
tar_file = Gem::Package::TarWriter.new(tar_file_stream)
|
82
|
+
Zip::File.open_buffer(package.stream) do |zip_file|
|
83
|
+
copy_archive(zip_file, tar_file, File.basename(PACKAGE_PATH))
|
84
|
+
tar_file.close
|
85
|
+
end
|
86
|
+
|
87
|
+
tar_file_stream.seek(0)
|
88
|
+
tar_file_stream
|
89
|
+
end
|
90
|
+
|
91
|
+
# Copies every entry from the zip archive to the tar archive, adding the optional prefix to the
|
92
|
+
# start of each file name.
|
93
|
+
#
|
94
|
+
# @param [Zip::File] zip_file The zip file to read from.
|
95
|
+
# @param [Gem::Package::TarWriter] tar_file The tar file to write to.
|
96
|
+
# @param [String] prefix The prefix to add to every file name in the tar.
|
97
|
+
def copy_archive(zip_file, tar_file, prefix = nil)
|
98
|
+
zip_file.each do |entry|
|
99
|
+
next unless entry.file?
|
100
|
+
|
101
|
+
zip_entry_stream = entry.get_input_stream
|
102
|
+
new_entry_name = prefix ? File.join(prefix, entry.name) : entry.name
|
103
|
+
tar_file.add_file(new_entry_name, 0664) do |tar_entry_stream|
|
104
|
+
IO.copy_stream(zip_entry_stream, tar_entry_stream)
|
105
|
+
end
|
106
|
+
|
107
|
+
zip_entry_stream.close
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def execute_package(container)
|
112
|
+
container.start!
|
113
|
+
container.wait
|
114
|
+
end
|
115
|
+
|
116
|
+
def extract_result(container)
|
117
|
+
logs = container.logs(stdout: true, stderr: true)
|
118
|
+
|
119
|
+
_, stdout, stderr = Coursemology::Evaluator::Utils.parse_docker_stream(logs)
|
120
|
+
Result.new(stdout, stderr, extract_test_report(container), container.exit_code)
|
121
|
+
end
|
122
|
+
|
123
|
+
def extract_test_report(container)
|
124
|
+
stream = extract_test_report_archive(container)
|
125
|
+
|
126
|
+
tar_file = Gem::Package::TarReader.new(stream)
|
127
|
+
tar_file.each do |file|
|
128
|
+
return file.read
|
129
|
+
end
|
130
|
+
rescue Docker::Error::NotFoundError
|
131
|
+
return nil
|
132
|
+
end
|
133
|
+
|
134
|
+
# Extracts the test report from the container.
|
135
|
+
#
|
136
|
+
# @return [StringIO] The stream containing the archive, the pointer is reset to the start of the
|
137
|
+
# stream.
|
138
|
+
def extract_test_report_archive(container)
|
139
|
+
stream = StringIO.new
|
140
|
+
container.archive_out(REPORT_PATH) do |bytes|
|
141
|
+
stream.write(bytes)
|
142
|
+
end
|
143
|
+
|
144
|
+
stream.seek(0)
|
145
|
+
stream
|
146
|
+
end
|
147
|
+
|
148
|
+
def destroy_container(container)
|
149
|
+
container.delete
|
150
|
+
end
|
151
|
+
end
|
@@ -1,14 +1,14 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
# Adapter for StringIO for compatibility with RubyZip.
|
3
|
-
#
|
4
|
-
# StringIO does not inherit from IO, so RubyZip does not accept StringIO in place of IO.
|
5
|
-
class Coursemology::Evaluator::StringIO < ::StringIO
|
6
|
-
def is_a?(klass)
|
7
|
-
klass == IO || super
|
8
|
-
end
|
9
|
-
|
10
|
-
# RubyZip assumes all IO objects respond to path.
|
11
|
-
def path
|
12
|
-
self
|
13
|
-
end
|
14
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# Adapter for StringIO for compatibility with RubyZip.
|
3
|
+
#
|
4
|
+
# StringIO does not inherit from IO, so RubyZip does not accept StringIO in place of IO.
|
5
|
+
class Coursemology::Evaluator::StringIO < ::StringIO
|
6
|
+
def is_a?(klass)
|
7
|
+
klass == IO || super
|
8
|
+
end
|
9
|
+
|
10
|
+
# RubyZip assumes all IO objects respond to path.
|
11
|
+
def path
|
12
|
+
self
|
13
|
+
end
|
14
|
+
end
|
@@ -1,42 +1,42 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
module Coursemology::Evaluator::Utils
|
3
|
-
# Represents one block of the Docker Attach protocol.
|
4
|
-
DockerAttachBlock = Struct.new(:stream, :length, :bytes)
|
5
|
-
|
6
|
-
# Parses a Docker +attach+ protocol stream into its constituent protocols.
|
7
|
-
#
|
8
|
-
# See https://docs.docker.com/engine/reference/api/docker_remote_api_v1.19/#attach-to-a-container.
|
9
|
-
#
|
10
|
-
# This drops all blocks belonging to streams other than STDIN, STDOUT, or STDERR.
|
11
|
-
#
|
12
|
-
# @param [String] string The input stream to parse.
|
13
|
-
# @return [Array<(String, String, String)>] The stdin, stdout, and stderr output.
|
14
|
-
def self.parse_docker_stream(string)
|
15
|
-
result = [''.dup, ''.dup, ''.dup]
|
16
|
-
stream = StringIO.new(string)
|
17
|
-
|
18
|
-
while (block = parse_docker_stream_read_block(stream))
|
19
|
-
next if block.stream >= result.length
|
20
|
-
result[block.stream] << block.bytes
|
21
|
-
end
|
22
|
-
|
23
|
-
stream.close
|
24
|
-
result
|
25
|
-
end
|
26
|
-
|
27
|
-
# Reads a block from the given stream, and parses it according to the Docker +attach+ protocol.
|
28
|
-
#
|
29
|
-
# @param [IO] stream The stream to read.
|
30
|
-
# @raise [IOError] If the stream is corrupt.
|
31
|
-
# @return [DockerAttachBlock] If there is data in the stream.
|
32
|
-
# @return [nil] If there is no data left in the stream.
|
33
|
-
def self.parse_docker_stream_read_block(stream)
|
34
|
-
header = stream.read(8)
|
35
|
-
return nil if header.blank?
|
36
|
-
fail IOError unless header.length == 8
|
37
|
-
|
38
|
-
console_stream, _, _, _, length = header.unpack('C4N')
|
39
|
-
DockerAttachBlock.new(console_stream, length, stream.read(length))
|
40
|
-
end
|
41
|
-
private_class_method :parse_docker_stream_read_block
|
42
|
-
end
|
1
|
+
# frozen_string_literal: true
|
2
|
+
module Coursemology::Evaluator::Utils
|
3
|
+
# Represents one block of the Docker Attach protocol.
|
4
|
+
DockerAttachBlock = Struct.new(:stream, :length, :bytes)
|
5
|
+
|
6
|
+
# Parses a Docker +attach+ protocol stream into its constituent protocols.
|
7
|
+
#
|
8
|
+
# See https://docs.docker.com/engine/reference/api/docker_remote_api_v1.19/#attach-to-a-container.
|
9
|
+
#
|
10
|
+
# This drops all blocks belonging to streams other than STDIN, STDOUT, or STDERR.
|
11
|
+
#
|
12
|
+
# @param [String] string The input stream to parse.
|
13
|
+
# @return [Array<(String, String, String)>] The stdin, stdout, and stderr output.
|
14
|
+
def self.parse_docker_stream(string)
|
15
|
+
result = [''.dup, ''.dup, ''.dup]
|
16
|
+
stream = StringIO.new(string)
|
17
|
+
|
18
|
+
while (block = parse_docker_stream_read_block(stream))
|
19
|
+
next if block.stream >= result.length
|
20
|
+
result[block.stream] << block.bytes
|
21
|
+
end
|
22
|
+
|
23
|
+
stream.close
|
24
|
+
result
|
25
|
+
end
|
26
|
+
|
27
|
+
# Reads a block from the given stream, and parses it according to the Docker +attach+ protocol.
|
28
|
+
#
|
29
|
+
# @param [IO] stream The stream to read.
|
30
|
+
# @raise [IOError] If the stream is corrupt.
|
31
|
+
# @return [DockerAttachBlock] If there is data in the stream.
|
32
|
+
# @return [nil] If there is no data left in the stream.
|
33
|
+
def self.parse_docker_stream_read_block(stream)
|
34
|
+
header = stream.read(8)
|
35
|
+
return nil if header.blank?
|
36
|
+
fail IOError unless header.length == 8
|
37
|
+
|
38
|
+
console_stream, _, _, _, length = header.unpack('C4N')
|
39
|
+
DockerAttachBlock.new(console_stream, length, stream.read(length))
|
40
|
+
end
|
41
|
+
private_class_method :parse_docker_stream_read_block
|
42
|
+
end
|