coursemology-evaluator 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/.env +3 -3
  3. data/.gitignore +23 -22
  4. data/.hound.yml +8 -8
  5. data/.idea/Coursemology Evaluator.iml +8 -8
  6. data/.rspec +2 -2
  7. data/.rubocop.unhound.yml +109 -109
  8. data/.rubocop.yml +46 -46
  9. data/.travis.yml +17 -17
  10. data/Gemfile +4 -4
  11. data/Procfile +1 -1
  12. data/README.md +29 -29
  13. data/Rakefile +6 -6
  14. data/bin/evaluator +5 -5
  15. data/coursemology-evaluator.gemspec +37 -37
  16. data/lib/coursemology/evaluator.rb +36 -36
  17. data/lib/coursemology/evaluator/cli.rb +52 -52
  18. data/lib/coursemology/evaluator/client.rb +81 -75
  19. data/lib/coursemology/evaluator/docker_container.rb +59 -59
  20. data/lib/coursemology/evaluator/logging.rb +12 -12
  21. data/lib/coursemology/evaluator/logging/client_log_subscriber.rb +25 -25
  22. data/lib/coursemology/evaluator/logging/docker_log_subscriber.rb +18 -18
  23. data/lib/coursemology/evaluator/models.rb +7 -7
  24. data/lib/coursemology/evaluator/models/base.rb +50 -50
  25. data/lib/coursemology/evaluator/models/programming_evaluation.rb +55 -67
  26. data/lib/coursemology/evaluator/models/programming_evaluation/package.rb +12 -12
  27. data/lib/coursemology/evaluator/services.rb +6 -6
  28. data/lib/coursemology/evaluator/services/evaluate_programming_package_service.rb +151 -151
  29. data/lib/coursemology/evaluator/string_io.rb +14 -14
  30. data/lib/coursemology/evaluator/utils.rb +42 -42
  31. data/lib/coursemology/evaluator/version.rb +5 -5
  32. data/lib/coursemology/polyglot/extensions.rb +3 -3
  33. data/lib/coursemology/polyglot/extensions/language.rb +24 -24
  34. metadata +15 -4
  35. data/Gemfile.lock +0 -114
@@ -1,12 +1,12 @@
1
- # frozen_string_literal: true
2
- class Coursemology::Evaluator::Models::ProgrammingEvaluation::Package
3
- # The stream comprising the package.
4
- attr_reader :stream
5
-
6
- # Constructs a new Package.
7
- #
8
- # @param [IO] stream The stream comprising the package.
9
- def initialize(stream)
10
- @stream = stream
11
- end
12
- end
1
+ # frozen_string_literal: true
2
+ class Coursemology::Evaluator::Models::ProgrammingEvaluation::Package
3
+ # The stream comprising the package.
4
+ attr_reader :stream
5
+
6
+ # Constructs a new Package.
7
+ #
8
+ # @param [IO] stream The stream comprising the package.
9
+ def initialize(stream)
10
+ @stream = stream
11
+ end
12
+ end
@@ -1,6 +1,6 @@
1
- # frozen_string_literal: true
2
- module Coursemology::Evaluator::Services
3
- extend ActiveSupport::Autoload
4
-
5
- autoload :EvaluateProgrammingPackageService
6
- end
1
+ # frozen_string_literal: true
2
+ module Coursemology::Evaluator::Services
3
+ extend ActiveSupport::Autoload
4
+
5
+ autoload :EvaluateProgrammingPackageService
6
+ end
@@ -1,151 +1,151 @@
1
- # frozen_string_literal: true
2
- class Coursemology::Evaluator::Services::EvaluateProgrammingPackageService
3
- Result = Struct.new(:stdout, :stderr, :test_report, :exit_code)
4
-
5
- # The path to the Coursemology user home directory.
6
- HOME_PATH = '/home/coursemology'.freeze
7
-
8
- # The path to where the package will be extracted.
9
- PACKAGE_PATH = File.join(HOME_PATH, 'package')
10
-
11
- # The path to where the test report will be at.
12
- REPORT_PATH = File.join(PACKAGE_PATH, 'report.xml')
13
-
14
- # The ratio to multiply the memory limits from our evaluation to the container by.
15
- MEMORY_LIMIT_RATIO = 1.megabyte / 1.kilobyte
16
-
17
- # Executes the given package in a container.
18
- #
19
- # @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
20
- # from the server.
21
- # @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result] The
22
- # result of the evaluation.
23
- def self.execute(evaluation)
24
- new(evaluation).send(:execute)
25
- end
26
-
27
- # Creates a new service object.
28
- def initialize(evaluation)
29
- @evaluation = evaluation
30
- @package = evaluation.package
31
- end
32
-
33
- private
34
-
35
- # Evaluates the package.
36
- #
37
- # @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result]
38
- def execute
39
- container = create_container(@evaluation.language.class.docker_image)
40
- copy_package(container)
41
- execute_package(container)
42
-
43
- extract_result(container)
44
- ensure
45
- destroy_container(container) if container
46
- end
47
-
48
- def create_container(image)
49
- image_identifier = "coursemology/evaluator-image-#{image}"
50
- Coursemology::Evaluator::DockerContainer.create(image_identifier, argv: container_arguments)
51
- end
52
-
53
- def container_arguments
54
- result = []
55
- result.push("-c#{@evaluation.time_limit}") if @evaluation.time_limit
56
- result.push("-m#{@evaluation.memory_limit * MEMORY_LIMIT_RATIO}") if @evaluation.memory_limit
57
-
58
- result
59
- end
60
-
61
- # Copies the contents of the package to the container.
62
- #
63
- # @param [Docker::Container] container The container to copy the package into.
64
- def copy_package(container)
65
- tar = tar_package(@package)
66
- container.archive_in_stream(HOME_PATH) do
67
- tar.read(Excon.defaults[:chunk_size]).to_s
68
- end
69
- end
70
-
71
- # Converts the zip package into a tar package for the container.
72
- #
73
- # This also adds an additional +package+ directory to the start of the path, following tar
74
- # convention.
75
- #
76
- # @param [Coursemology::Evaluator::Models::ProgrammingEvaluation::Package] package The package
77
- # to convert to a tar.
78
- # @return [IO] A stream containing the tar.
79
- def tar_package(package)
80
- tar_file_stream = StringIO.new
81
- tar_file = Gem::Package::TarWriter.new(tar_file_stream)
82
- Zip::File.open_buffer(package.stream) do |zip_file|
83
- copy_archive(zip_file, tar_file, File.basename(PACKAGE_PATH))
84
- tar_file.close
85
- end
86
-
87
- tar_file_stream.seek(0)
88
- tar_file_stream
89
- end
90
-
91
- # Copies every entry from the zip archive to the tar archive, adding the optional prefix to the
92
- # start of each file name.
93
- #
94
- # @param [Zip::File] zip_file The zip file to read from.
95
- # @param [Gem::Package::TarWriter] tar_file The tar file to write to.
96
- # @param [String] prefix The prefix to add to every file name in the tar.
97
- def copy_archive(zip_file, tar_file, prefix = nil)
98
- zip_file.each do |entry|
99
- next unless entry.file?
100
-
101
- zip_entry_stream = entry.get_input_stream
102
- new_entry_name = prefix ? File.join(prefix, entry.name) : entry.name
103
- tar_file.add_file(new_entry_name, 0664) do |tar_entry_stream|
104
- IO.copy_stream(zip_entry_stream, tar_entry_stream)
105
- end
106
-
107
- zip_entry_stream.close
108
- end
109
- end
110
-
111
- def execute_package(container)
112
- container.start!
113
- container.wait
114
- end
115
-
116
- def extract_result(container)
117
- logs = container.logs(stdout: true, stderr: true)
118
-
119
- _, stdout, stderr = Coursemology::Evaluator::Utils.parse_docker_stream(logs)
120
- Result.new(stdout, stderr, extract_test_report(container), container.exit_code)
121
- end
122
-
123
- def extract_test_report(container)
124
- stream = extract_test_report_archive(container)
125
-
126
- tar_file = Gem::Package::TarReader.new(stream)
127
- tar_file.each do |file|
128
- return file.read
129
- end
130
- rescue Docker::Error::NotFoundError
131
- return nil
132
- end
133
-
134
- # Extracts the test report from the container.
135
- #
136
- # @return [StringIO] The stream containing the archive, the pointer is reset to the start of the
137
- # stream.
138
- def extract_test_report_archive(container)
139
- stream = StringIO.new
140
- container.archive_out(REPORT_PATH) do |bytes|
141
- stream.write(bytes)
142
- end
143
-
144
- stream.seek(0)
145
- stream
146
- end
147
-
148
- def destroy_container(container)
149
- container.delete
150
- end
151
- end
1
+ # frozen_string_literal: true
2
+ class Coursemology::Evaluator::Services::EvaluateProgrammingPackageService
3
+ Result = Struct.new(:stdout, :stderr, :test_report, :exit_code)
4
+
5
+ # The path to the Coursemology user home directory.
6
+ HOME_PATH = '/home/coursemology'.freeze
7
+
8
+ # The path to where the package will be extracted.
9
+ PACKAGE_PATH = File.join(HOME_PATH, 'package')
10
+
11
+ # The path to where the test report will be at.
12
+ REPORT_PATH = File.join(PACKAGE_PATH, 'report.xml')
13
+
14
+ # The ratio to multiply the memory limits from our evaluation to the container by.
15
+ MEMORY_LIMIT_RATIO = 1.megabyte / 1.kilobyte
16
+
17
+ # Executes the given package in a container.
18
+ #
19
+ # @param [Coursemology::Evaluator::Models::ProgrammingEvaluation] evaluation The evaluation
20
+ # from the server.
21
+ # @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result] The
22
+ # result of the evaluation.
23
+ def self.execute(evaluation)
24
+ new(evaluation).send(:execute)
25
+ end
26
+
27
+ # Creates a new service object.
28
+ def initialize(evaluation)
29
+ @evaluation = evaluation
30
+ @package = evaluation.package
31
+ end
32
+
33
+ private
34
+
35
+ # Evaluates the package.
36
+ #
37
+ # @return [Coursemology::Evaluator::Services::EvaluateProgrammingPackageService::Result]
38
+ def execute
39
+ container = create_container(@evaluation.language.class.docker_image)
40
+ copy_package(container)
41
+ execute_package(container)
42
+
43
+ extract_result(container)
44
+ ensure
45
+ destroy_container(container) if container
46
+ end
47
+
48
+ def create_container(image)
49
+ image_identifier = "coursemology/evaluator-image-#{image}"
50
+ Coursemology::Evaluator::DockerContainer.create(image_identifier, argv: container_arguments)
51
+ end
52
+
53
+ def container_arguments
54
+ result = []
55
+ result.push("-c#{@evaluation.time_limit}") if @evaluation.time_limit
56
+ result.push("-m#{@evaluation.memory_limit * MEMORY_LIMIT_RATIO}") if @evaluation.memory_limit
57
+
58
+ result
59
+ end
60
+
61
+ # Copies the contents of the package to the container.
62
+ #
63
+ # @param [Docker::Container] container The container to copy the package into.
64
+ def copy_package(container)
65
+ tar = tar_package(@package)
66
+ container.archive_in_stream(HOME_PATH) do
67
+ tar.read(Excon.defaults[:chunk_size]).to_s
68
+ end
69
+ end
70
+
71
+ # Converts the zip package into a tar package for the container.
72
+ #
73
+ # This also adds an additional +package+ directory to the start of the path, following tar
74
+ # convention.
75
+ #
76
+ # @param [Coursemology::Evaluator::Models::ProgrammingEvaluation::Package] package The package
77
+ # to convert to a tar.
78
+ # @return [IO] A stream containing the tar.
79
+ def tar_package(package)
80
+ tar_file_stream = StringIO.new
81
+ tar_file = Gem::Package::TarWriter.new(tar_file_stream)
82
+ Zip::File.open_buffer(package.stream) do |zip_file|
83
+ copy_archive(zip_file, tar_file, File.basename(PACKAGE_PATH))
84
+ tar_file.close
85
+ end
86
+
87
+ tar_file_stream.seek(0)
88
+ tar_file_stream
89
+ end
90
+
91
+ # Copies every entry from the zip archive to the tar archive, adding the optional prefix to the
92
+ # start of each file name.
93
+ #
94
+ # @param [Zip::File] zip_file The zip file to read from.
95
+ # @param [Gem::Package::TarWriter] tar_file The tar file to write to.
96
+ # @param [String] prefix The prefix to add to every file name in the tar.
97
+ def copy_archive(zip_file, tar_file, prefix = nil)
98
+ zip_file.each do |entry|
99
+ next unless entry.file?
100
+
101
+ zip_entry_stream = entry.get_input_stream
102
+ new_entry_name = prefix ? File.join(prefix, entry.name) : entry.name
103
+ tar_file.add_file(new_entry_name, 0664) do |tar_entry_stream|
104
+ IO.copy_stream(zip_entry_stream, tar_entry_stream)
105
+ end
106
+
107
+ zip_entry_stream.close
108
+ end
109
+ end
110
+
111
+ def execute_package(container)
112
+ container.start!
113
+ container.wait
114
+ end
115
+
116
+ def extract_result(container)
117
+ logs = container.logs(stdout: true, stderr: true)
118
+
119
+ _, stdout, stderr = Coursemology::Evaluator::Utils.parse_docker_stream(logs)
120
+ Result.new(stdout, stderr, extract_test_report(container), container.exit_code)
121
+ end
122
+
123
+ def extract_test_report(container)
124
+ stream = extract_test_report_archive(container)
125
+
126
+ tar_file = Gem::Package::TarReader.new(stream)
127
+ tar_file.each do |file|
128
+ return file.read
129
+ end
130
+ rescue Docker::Error::NotFoundError
131
+ return nil
132
+ end
133
+
134
+ # Extracts the test report from the container.
135
+ #
136
+ # @return [StringIO] The stream containing the archive, the pointer is reset to the start of the
137
+ # stream.
138
+ def extract_test_report_archive(container)
139
+ stream = StringIO.new
140
+ container.archive_out(REPORT_PATH) do |bytes|
141
+ stream.write(bytes)
142
+ end
143
+
144
+ stream.seek(0)
145
+ stream
146
+ end
147
+
148
+ def destroy_container(container)
149
+ container.delete
150
+ end
151
+ end
@@ -1,14 +1,14 @@
1
- # frozen_string_literal: true
2
- # Adapter for StringIO for compatibility with RubyZip.
3
- #
4
- # StringIO does not inherit from IO, so RubyZip does not accept StringIO in place of IO.
5
- class Coursemology::Evaluator::StringIO < ::StringIO
6
- def is_a?(klass)
7
- klass == IO || super
8
- end
9
-
10
- # RubyZip assumes all IO objects respond to path.
11
- def path
12
- self
13
- end
14
- end
1
+ # frozen_string_literal: true
2
+ # Adapter for StringIO for compatibility with RubyZip.
3
+ #
4
+ # StringIO does not inherit from IO, so RubyZip does not accept StringIO in place of IO.
5
+ class Coursemology::Evaluator::StringIO < ::StringIO
6
+ def is_a?(klass)
7
+ klass == IO || super
8
+ end
9
+
10
+ # RubyZip assumes all IO objects respond to path.
11
+ def path
12
+ self
13
+ end
14
+ end
@@ -1,42 +1,42 @@
1
- # frozen_string_literal: true
2
- module Coursemology::Evaluator::Utils
3
- # Represents one block of the Docker Attach protocol.
4
- DockerAttachBlock = Struct.new(:stream, :length, :bytes)
5
-
6
- # Parses a Docker +attach+ protocol stream into its constituent protocols.
7
- #
8
- # See https://docs.docker.com/engine/reference/api/docker_remote_api_v1.19/#attach-to-a-container.
9
- #
10
- # This drops all blocks belonging to streams other than STDIN, STDOUT, or STDERR.
11
- #
12
- # @param [String] string The input stream to parse.
13
- # @return [Array<(String, String, String)>] The stdin, stdout, and stderr output.
14
- def self.parse_docker_stream(string)
15
- result = [''.dup, ''.dup, ''.dup]
16
- stream = StringIO.new(string)
17
-
18
- while (block = parse_docker_stream_read_block(stream))
19
- next if block.stream >= result.length
20
- result[block.stream] << block.bytes
21
- end
22
-
23
- stream.close
24
- result
25
- end
26
-
27
- # Reads a block from the given stream, and parses it according to the Docker +attach+ protocol.
28
- #
29
- # @param [IO] stream The stream to read.
30
- # @raise [IOError] If the stream is corrupt.
31
- # @return [DockerAttachBlock] If there is data in the stream.
32
- # @return [nil] If there is no data left in the stream.
33
- def self.parse_docker_stream_read_block(stream)
34
- header = stream.read(8)
35
- return nil if header.blank?
36
- fail IOError unless header.length == 8
37
-
38
- console_stream, _, _, _, length = header.unpack('C4N')
39
- DockerAttachBlock.new(console_stream, length, stream.read(length))
40
- end
41
- private_class_method :parse_docker_stream_read_block
42
- end
1
+ # frozen_string_literal: true
2
+ module Coursemology::Evaluator::Utils
3
+ # Represents one block of the Docker Attach protocol.
4
+ DockerAttachBlock = Struct.new(:stream, :length, :bytes)
5
+
6
+ # Parses a Docker +attach+ protocol stream into its constituent protocols.
7
+ #
8
+ # See https://docs.docker.com/engine/reference/api/docker_remote_api_v1.19/#attach-to-a-container.
9
+ #
10
+ # This drops all blocks belonging to streams other than STDIN, STDOUT, or STDERR.
11
+ #
12
+ # @param [String] string The input stream to parse.
13
+ # @return [Array<(String, String, String)>] The stdin, stdout, and stderr output.
14
+ def self.parse_docker_stream(string)
15
+ result = [''.dup, ''.dup, ''.dup]
16
+ stream = StringIO.new(string)
17
+
18
+ while (block = parse_docker_stream_read_block(stream))
19
+ next if block.stream >= result.length
20
+ result[block.stream] << block.bytes
21
+ end
22
+
23
+ stream.close
24
+ result
25
+ end
26
+
27
+ # Reads a block from the given stream, and parses it according to the Docker +attach+ protocol.
28
+ #
29
+ # @param [IO] stream The stream to read.
30
+ # @raise [IOError] If the stream is corrupt.
31
+ # @return [DockerAttachBlock] If there is data in the stream.
32
+ # @return [nil] If there is no data left in the stream.
33
+ def self.parse_docker_stream_read_block(stream)
34
+ header = stream.read(8)
35
+ return nil if header.blank?
36
+ fail IOError unless header.length == 8
37
+
38
+ console_stream, _, _, _, length = header.unpack('C4N')
39
+ DockerAttachBlock.new(console_stream, length, stream.read(length))
40
+ end
41
+ private_class_method :parse_docker_stream_read_block
42
+ end