proformaxml 0.10.0

data/assets/schemas/proforma-java-checkstyle.xsd

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ns1="urn:proforma:tests:java-checkstyle:v1.1" targetNamespace="urn:proforma:tests:java-checkstyle:v1.1" elementFormDefault="qualified" attributeFormDefault="unqualified">
+    <xs:element name="java-checkstyle">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element name="max-checkstyle-warnings" type="xs:string"/>
+            </xs:sequence>
+            <xs:attribute name="version" type="xs:string" use="required"/>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>

data/assets/schemas/proforma-regexptest.xsd

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- This is meant for blackbox tests of the output of a program. For regexp tests of 
+sourcecode something like checkstyle can be used.-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ns1="urn:proforma:tests:regexptest:v0.9"
+    targetNamespace="urn:proforma:tests:regexptest:v0.9" elementFormDefault="qualified"
+    attributeFormDefault="unqualified">
+    <xs:element name="regexptest">
+        <xs:complexType>
+            <xs:sequence>
+                <!-- entry-point is a file or a main-class needed to start a program -->
+                <xs:element name="entry-point" type="xs:string"/>
+                <!-- parameter: optional list of command line arguments -->
+                <xs:element name="parameter" minOccurs="0">
+                    <xs:simpleType>
+                        <xs:list itemType="xs:string"/>
+                    </xs:simpleType>
+                </xs:element>
+                <xs:element name="regular-expressions">
+                    <xs:complexType>
+                        <!-- choice means: regexp-allow OR regexp-disallow -->
+                        <xs:choice>
+                            <xs:sequence>
+                                <!-- regular expression for allowed program outputs -->
+                                <xs:element name="regexp-allow" type="ns1:regexpType" maxOccurs="unbounded"/>
+                                <!-- regular expression for not allowed program outputs -->
+                                <xs:element name="regexp-disallow" type="ns1:regexpType" minOccurs="0" maxOccurs="unbounded"/>
+                            </xs:sequence>
+                            <xs:element name="regexp-disallow" type="ns1:regexpType" maxOccurs="unbounded"/>
+                        </xs:choice>
+                    </xs:complexType>
+                </xs:element>
+            </xs:sequence>
+        </xs:complexType>
+    </xs:element>
+
+    <xs:complexType name="regexpType">
+        <xs:simpleContent>
+            <xs:extension base="xs:string">
+                <xs:attributeGroup ref="ns1:regexp-flags"/>
+            </xs:extension>
+        </xs:simpleContent>
+    </xs:complexType>
+    <!-- attributes to define, which regexp-flags are applied to a specified regular expression -->
+    <xs:attributeGroup name="regexp-flags">
+        <xs:attribute name="case-insensitive" type="xs:boolean" default="false"/>
+        <xs:attribute name="dotall" type="xs:boolean" default="false"/>
+        <xs:attribute name="multiline" type="xs:boolean" default="false"/>
+        <xs:attribute name="free-spacing" type="xs:boolean" default="false"/>
+    </xs:attributeGroup>
+</xs:schema>

data/assets/schemas/proforma-unittest.xsd

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ns1="urn:proforma:tests:unittest:v1.1" targetNamespace="urn:proforma:tests:unittest:v1.1" elementFormDefault="qualified" attributeFormDefault="unqualified">
+    <xs:element name="unittest">
+        <xs:complexType>
+            <xs:sequence>
+                <xs:element name="entry-point" type="xs:string" maxOccurs="unbounded"/>
+            </xs:sequence>
+            <xs:attribute name="framework" type="xs:string" use="required"/>
+            <xs:attribute name="version" type="xs:string" use="required"/>
+        </xs:complexType>
+    </xs:element>
+</xs:schema>

data/lib/proformaxml/errors.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  class ProformaError < StandardError; end
+
+  class PostGenerateValidationError < ProformaError; end
+
+  class PreImportValidationError < ProformaError; end
+end

data/lib/proformaxml/helpers/export_helpers.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  module Helpers
+    module ExportHelpers
+      def attach_file(xml, file)
+        if file.embed?
+          embed_file(file, xml)
+        else
+          xml.send "attached-#{file.binary ? 'bin' : 'txt'}-file", file.filename
+        end
+      end
+
+      def embed_file(file, xml)
+        if file.binary
+          xml.send :'embedded-bin-file', {filename: file.filename}, Base64.encode64(file.content)
+        else
+          xml.send :'embedded-txt-file', {filename: file.filename}, file.content
+        end
+      end
+
+      def add_description_to_xml(xml, description)
+        xml.send(:description, description) if description.present?
+      end
+
+      def add_test_configuration(xml, test)
+        xml.send(:'test-configuration') do
+          add_filerefs(xml, test) if test.files
+          add_configuration(xml, test.configuration) unless test.configuration.nil?
+          if test.meta_data
+            xml.send(:'test-meta-data') do
+              meta_data(xml, test.meta_data)
+            end
+          end
+        end
+      end
+
+      def inner_meta_data(xml, namespace, data)
+        data.each do |key, value|
+          case value.class.name
+            when 'Hash'
+              # underscore is used to disambiguate tag names from ruby methods
+              xml[namespace].send("#{key}_") do |meta_data_xml|
+                inner_meta_data(meta_data_xml, namespace, value)
+              end
+            else
+              xml[namespace].send("#{key}_", value)
+          end
+        end
+      end
+
+      def meta_data(xml, meta_data)
+        meta_data.each do |namespace, data|
+          inner_meta_data(xml, namespace, data)
+        end
+      end
+
+      def add_configuration(xml, configuration)
+        xml_snippet = Dachsfisch::JSON2XMLConverter.perform(json: configuration.to_json)
+        configuration.flat_map {|_, val| val['@xmlns'].to_a }.uniq.each do |namespace|
+          xml.doc.root.add_namespace(namespace[0], namespace[1])
+        end
+
+        xml << xml_snippet
+      end
+
+      def add_namespaces_to_header(header, custom_namespaces)
+        custom_namespaces.each do |namespace|
+          header["xmlns:#{namespace[:prefix]}"] = namespace[:uri]
+        end
+      end
+
+      def add_parent_uuid_and_lang_to_header(header)
+        header['lang'] = @task.language if @task.language.present?
+        header['parent-uuid'] = @task.parent_uuid if @task.parent_uuid.present?
+      end
+    end
+  end
+end

data/lib/proformaxml/helpers/import_helpers.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  module Helpers
+    module ImportHelpers
+      CONFIGURATION_NODES = %w[filerefs timeout externalresourcerefs test-meta-data].freeze
+
+      def set_hash_value_if_present(hash:, name:, attributes: nil, value_overwrite: nil)
+        raise unless attributes || value_overwrite
+
+        value = value_overwrite || attributes[name.to_s]&.value
+        hash[name.underscore.to_sym] = value if value.present?
+      end
+
+      def set_value_from_xml(object:, node:, name:, attribute: false, check_presence: true)
+        value = value_from_node(name, node, attribute)
+        return if check_presence && value.blank?
+
+        set_value(object:, name: (name.is_a?(Array) ? name[1] : name).underscore, value:)
+      end
+
+      def set_value(object:, name:, value:)
+        if object.is_a? Hash
+          object[name] = value
+        else
+          object.send("#{name}=", value)
+        end
+      end
+
+      def embedded_file_attributes(attributes, file_tag)
+        shared = shared_file_attributes(attributes, file_tag)
+        shared.merge(
+          content: content_from_file_tag(file_tag, shared[:binary])
+        ).tap do |hash|
+          hash[:filename] = file_tag.attributes['filename']&.value if file_tag.attributes['filename']&.value.present?
+        end
+      end
+
+      def attached_file_attributes(attributes, file_tag)
+        filename = file_tag.text
+        shared_file_attributes(attributes, file_tag).merge(filename:,
+          content: filestring_from_zip(filename))
+      end
+
+      def shared_file_attributes(attributes, file_tag)
+        {
+          id: attributes['id']&.value,
+          used_by_grader: attributes['used-by-grader']&.value == 'true',
+          visible: attributes['visible']&.value,
+          binary: file_tag.name.include?('-bin-file'),
+        }.tap do |hash|
+          set_hash_value_if_present(hash:, name: 'usage-by-lms', attributes:)
+          set_value_from_xml(object: hash, node: file_tag.parent, name: 'internal-description')
+          set_hash_value_if_present(hash:, name: 'mimetype', attributes:)
+        end
+      end
+
+      def add_test_configuration(test, test_node)
+        test_configuration_node = test_node.xpath('xmlns:test-configuration')
+        test.files = test_files_from_test_configuration(test_configuration_node)
+        test.configuration = extra_configuration_from_test_configuration(test_configuration_node)
+        meta_data_node = test_node.xpath('xmlns:test-configuration').xpath('xmlns:test-meta-data')
+        test.meta_data = meta_data(meta_data_node, use_namespace: true) if meta_data_node.present?
+      end
+
+      def extra_configuration_from_test_configuration(test_configuration_node)
+        configuration_any_nodes = test_configuration_node.children.reject {|c| CONFIGURATION_NODES.include? c.name }
+        return nil if configuration_any_nodes.empty?
+
+        {}.tap do |config_hash|
+          configuration_any_nodes.each do |config_node|
+            config_hash.merge! convert_xml_node_to_json(config_node)
+          end
+        end
+      end
+
+      def test_files_from_test_configuration(test_configuration_node)
+        files_from_filerefs(test_configuration_node.search('filerefs'))
+      end
+
+      def meta_data(any_data_node, use_namespace: false)
+        # use_namespace forces the use of the namespace as hash key - it should only be used at the entry of the recursion
+        {}.tap do |any_data|
+          any_data_node.children.each do |node|
+            key = (use_namespace ? node.namespace.prefix : any_data_node.name).to_sym
+            any_data[key] = if node.node_type == Nokogiri::XML::Node::TEXT_NODE
+                              node.text
+                            else
+                              # preserve any existing data in the nested hash
+                              (any_data[key] || {}).merge meta_data(node)
+                            end
+          end
+        end
+      end
+
+      private
+
+      def convert_xml_node_to_json(any_node)
+        xml_snippet = Nokogiri::XML::DocumentFragment.new(Nokogiri::XML::Document.new, any_node)
+        xml_snippet.children.first.add_namespace_definition(any_node.namespace.prefix, any_node.namespace.href)
+        JSON.parse(Dachsfisch::XML2JSONConverter.perform(xml: xml_snippet.to_xml))
+      end
+
+      def value_from_node(name, node, attribute)
+        xml_name = name.is_a?(Array) ? name[0] : name
+        attribute ? node.attribute(xml_name)&.value : node.xpath("xmlns:#{xml_name}").text
+      end
+
+      def content_from_file_tag(file_tag, binary)
+        binary ? Base64.decode64(file_tag.text) : file_tag.text
+      end
+    end
+  end
+end

data/lib/proformaxml/models/base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  class Base
+    include ActiveModel::AttributeAssignment
+
+    def initialize(attributes = {})
+      assign_attributes(attributes)
+    end
+  end
+end

data/lib/proformaxml/models/model_solution.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  class ModelSolution < Base
+    attr_accessor :id, :files, :description, :internal_description
+
+    def initialize(attributes = {})
+      super
+      self.files = [] if files.nil?
+    end
+  end
+end

data/lib/proformaxml/models/task.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'proformaxml/models/base'
+require 'proformaxml/models/task_file'
+require 'proformaxml/models/test'
+require 'proformaxml/models/model_solution'
+require 'proformaxml/errors'
+
+module ProformaXML
+  class Task < Base
+    attr_accessor :title, :description, :internal_description, :proglang, :uuid, :parent_uuid,
+      :language, :model_solutions, :files, :tests, :meta_data
+
+    def initialize(attributes = {})
+      super
+      self.files = [] if files.nil?
+      self.tests = [] if tests.nil?
+      self.model_solutions = [] if model_solutions.nil?
+      self.meta_data = {} if meta_data.nil?
+    end
+
+    def all_files
+      task_files = files
+      model_solution_files = model_solutions.map(&:files).filter(&:present?)
+      test_files = tests.map(&:files).filter(&:present?)
+      (task_files + model_solution_files + test_files).flatten.uniq
+    end
+  end
+end

data/lib/proformaxml/models/task_file.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  class TaskFile < Base
+    attr_accessor :id, :content, :filename, :used_by_grader, :visible, :usage_by_lms, :binary, :internal_description, :mimetype
+
+    def embed?
+      (content&.length || 0) < MAX_EMBEDDED_FILE_SIZE_KB * (2**10)
+    end
+  end
+end

data/lib/proformaxml/models/test.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module ProformaXML
+  class Test < Base
+    attr_accessor :id, :title, :description, :internal_description, :test_type, :files, :configuration, :meta_data
+
+    def initialize(attributes = {})
+      super
+      self.files = [] if files.nil?
+    end
+  end
+end

data/lib/proformaxml/services/exporter.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'proformaxml/helpers/export_helpers'
+
+module ProformaXML
+  class Exporter
+    include ProformaXML::Helpers::ExportHelpers
+
+    def initialize(task:, custom_namespaces: [], version: nil)
+      @files = {}
+      @task = task
+      @custom_namespaces = custom_namespaces
+      @version = version || SCHEMA_VERSIONS.first
+      add_placeholders if @version == '2.0'
+    end
+
+    def perform
+      builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml_task(xml)
+      end
+      xmldoc = builder.to_xml
+      doc = Nokogiri::XML(xmldoc)
+      errors = validate(doc)
+
+      raise PostGenerateValidationError.new(errors) if errors.any?
+
+      # File.open('../testfile.zip', 'wb') { |file| file.write(write_to_zip(xmldoc).string) }
+      write_to_zip(xmldoc)
+    end
+
+    private
+
+    def xml_task(xml)
+      xml.task(headers) do
+        xml.title @task.title
+        xml.description @task.description
+        add_internal_description_to_xml(xml, @task.internal_description)
+        xml.proglang({version: @task.proglang&.dig(:version)}, @task.proglang&.dig(:name))
+
+        add_objects_to_xml(xml)
+        add_meta_data(xml)
+      end
+    end
+
+    def add_internal_description_to_xml(xml, internal_description)
+      xml.send(:'internal-description', internal_description) if internal_description.present?
+    end
+
+    def add_meta_data(xml)
+      xml.send(:'meta-data') { meta_data(xml, @task.meta_data) }
+    end
+
+    def add_objects_to_xml(xml)
+      xml.files { files(xml) }
+      xml.send(:'model-solutions') { model_solutions(xml) } if @task.model_solutions.any? || @version == '2.0'
+      xml.tests { tests(xml) }
+    end
+
+    def files(xml)
+      @task.all_files.each do |file|
+        xml.file({
+          id: file.id, 'used-by-grader' => file.used_by_grader, visible: file.visible,
+          'usage-by-lms' => file.usage_by_lms, mimetype: file.mimetype
+        }.compact) do
+          attach_file(xml, file)
+          add_internal_description_to_xml(xml, file.internal_description)
+        end
+      end
+    end
+
+    def model_solutions(xml)
+      @task.model_solutions&.each do |model_solution|
+        xml.send(:'model-solution', id: model_solution.id) do
+          add_filerefs(xml, model_solution)
+          add_description_to_xml(xml, model_solution.description)
+          add_internal_description_to_xml(xml, model_solution.internal_description)
+        end
+      end
+    end
+
+    def add_filerefs(xml, object)
+      return unless object.files.any?
+
+      xml.filerefs do
+        object.files.each do |file|
+          xml.fileref(refid: file.id) {}
+        end
+      end
+    end
+
+    def tests(xml)
+      @task.tests&.each do |test|
+        xml.test(id: test.id) do
+          xml.title test.title
+          add_description_to_xml(xml, test.description)
+          add_internal_description_to_xml(xml, test.internal_description)
+          xml.send(:'test-type', test.test_type)
+          add_test_configuration(xml, test)
+        end
+      end
+    end
+
+    # ms-placeholder only necessary for version 2.0 where model-solutions were mandatory
+    def add_placeholders
+      return if @task.model_solutions&.any?
+
+      file = TaskFile.new(content: '', id: 'ms-placeholder-file', used_by_grader: false, visible: 'no', binary: false)
+      model_solution = ModelSolution.new(id: 'ms-placeholder', files: [file])
+      @task.model_solutions = [model_solution]
+    end
+
+    def headers
+      {
+        'xmlns' => "urn:proforma:v#{@version}",
+        'uuid' => @task.uuid,
+      }.tap do |header|
+        add_namespaces_to_header(header, @custom_namespaces)
+        add_parent_uuid_and_lang_to_header(header)
+      end
+    end
+
+    def validate(doc)
+      validator = ProformaXML::Validator.new doc, @version
+      validator.perform
+    end
+
+    def write_to_zip(xmldoc)
+      Zip::OutputStream.write_buffer do |zio|
+        zio.put_next_entry('task.xml')
+        zio.write xmldoc
+        @task.all_files.each do |file|
+          next if file.embed?
+
+          zio.put_next_entry(file.filename)
+          zio.write file.content
+        end
+      end
+    end
+  end
+end

data/lib/proformaxml/services/importer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/string'
+require 'proformaxml/helpers/import_helpers'
+
+module ProformaXML
+  class Importer
+    include ProformaXML::Helpers::ImportHelpers
+
+    def initialize(zip:, expected_version: nil)
+      @zip = zip
+      @expected_version = expected_version
+
+      xml = filestring_from_zip('task.xml')
+      raise PreImportValidationError if xml.nil?
+
+      @doc = Nokogiri::XML(xml, &:noblanks)
+      @task = Task.new
+    end
+
+    def perform
+      errors = validate
+
+      raise PreImportValidationError.new(errors) if errors.any?
+
+      @task_node = @doc.xpath('/xmlns:task')
+
+      set_data
+      {task: @task, custom_namespaces: @custom_namespaces}
+    end
+
+    private
+
+    def filestring_from_zip(filename)
+      Zip::File.open(@zip.path) do |zip_file|
+        return zip_file.glob(filename).first&.get_input_stream&.read
+      end
+    end
+
+    def set_data
+      set_namespaces
+      set_base_data
+      set_files
+      set_model_solutions
+      set_tests
+      set_meta_data
+    end
+
+    def set_namespaces
+      @custom_namespaces = @doc.namespaces.except('xmlns').map {|k, v| {prefix: k[6..], uri: v} }
+    end
+
+    def set_base_data
+      set_value_from_xml(object: @task, node: @task_node, name: 'title')
+      set_value_from_xml(object: @task, node: @task_node, name: 'description')
+      set_value_from_xml(object: @task, node: @task_node, name: 'internal-description')
+      set_proglang
+      set_value_from_xml(object: @task, node: @task_node, name: %w[lang language], attribute: true)
+      set_value_from_xml(object: @task, node: @task_node, name: 'parent-uuid', attribute: true)
+      set_value_from_xml(object: @task, node: @task_node, name: 'uuid', attribute: true)
+    end
+
+    def set_proglang
+      return if @task_node.xpath('xmlns:proglang').text.blank?
+
+      @task.proglang = {name: @task_node.xpath('xmlns:proglang').text,
+                        version: @task_node.xpath('xmlns:proglang').attribute('version').value.presence}.compact
+    end
+
+    def set_files
+      @task_node.search('files//file').each {|file_node| add_file file_node }
+    end
+
+    def set_tests
+      @task_node.search('tests//test').each {|test_node| add_test test_node }
+    end
+
+    def set_model_solutions
+      @task_node.search('model-solutions//model-solution').each do |model_solution_node|
+        add_model_solution model_solution_node
+      end
+    end
+
+    def set_meta_data
+      meta_data_node = @task_node.xpath('xmlns:meta-data')
+      @task.meta_data = meta_data(meta_data_node, use_namespace: true) if meta_data_node.text.present?
+    end
+
+    def add_model_solution(model_solution_node)
+      model_solution = ModelSolution.new
+      model_solution.id = model_solution_node.attributes['id'].value
+      model_solution.files = files_from_filerefs(model_solution_node.search('filerefs'))
+      set_value_from_xml(object: model_solution, node: model_solution_node, name: 'description')
+      set_value_from_xml(object: model_solution, node: model_solution_node, name: 'internal-description')
+      @task.model_solutions << model_solution unless model_solution.files.first&.id == 'ms-placeholder-file'
+    end
+
+    def add_file(file_node)
+      file_tag = file_node.children.first
+      file = nil
+      case file_tag.name
+        when /embedded-(bin|txt)-file/
+          file = TaskFile.new(embedded_file_attributes(file_node.attributes, file_tag))
+        when /attached-(bin|txt)-file/
+          file = TaskFile.new(attached_file_attributes(file_node.attributes, file_tag))
+      end
+      @task.files << file
+    end
+
+    def add_test(test_node)
+      test = Test.new
+      set_value_from_xml(object: test, node: test_node, name: 'id', attribute: true, check_presence: false)
+      set_value_from_xml(object: test, node: test_node, name: 'title', check_presence: false)
+      set_value_from_xml(object: test, node: test_node, name: 'description')
+      set_value_from_xml(object: test, node: test_node, name: 'internal-description')
+      set_value_from_xml(object: test, node: test_node, name: 'test-type', check_presence: false)
+      add_test_configuration(test, test_node)
+      @task.tests << test
+    end
+
+    def files_from_filerefs(filerefs_node)
+      [].tap do |files|
+        filerefs_node.search('fileref').each do |fileref_node|
+          fileref = fileref_node.attributes['refid'].value
+          files << @task.files.delete(@task.files.detect {|file| file.id == fileref })
+        end
+      end
+    end
+
+    def validate
+      validator = ProformaXML::Validator.new @doc, @expected_version
+      validator.perform
+    end
+  end
+end