atomic_assessments_import 0.3.0 → 0.5.0

data/lib/atomic_assessments_import/csv/converter.rb

@@ -3,9 +3,9 @@
 require "csv"
 require "active_support/core_ext/digest/uuid"
 
-require_relative "questions/question"
-require_relative "questions/multiple_choice"
-require_relative "utils"
+require_relative "../questions/question"
+require_relative "../questions/multiple_choice"
+require_relative "../utils"
 
 module AtomicAssessmentsImport
   module CSV

data/lib/atomic_assessments_import/exam_soft/chunker/heading_split_strategy.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "strategy"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      class HeadingSplitStrategy < Strategy
+        HEADING_PATTERN = /^h[1-6]$/i
+
+        def split(doc)
+          @header_nodes = []
+          chunks = []
+          current_chunk = []
+          found_first = false
+
+          doc.children.each do |node|
+            text = node.text.strip
+            next if text.empty? && !node.name.match?(/^(img|table|hr)$/i)
+
+            if node.name.match?(HEADING_PATTERN)
+              found_first = true
+              chunks << current_chunk unless current_chunk.empty?
+              current_chunk = [node]
+            elsif found_first
+              current_chunk << node
+            else
+              @header_nodes << node
+            end
+          end
+
+          chunks << current_chunk unless current_chunk.empty?
+          chunks.length >= 2 ? chunks : []
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/chunker/horizontal_rule_split_strategy.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "strategy"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      class HorizontalRuleSplitStrategy < Strategy
+        def split(doc)
+          @header_nodes = []
+          segments = []
+          current_segment = []
+
+          doc.children.each do |node|
+            text = node.text.strip
+
+            if node.name.match?(/^hr$/i)
+              segments << current_segment unless current_segment.empty?
+              current_segment = []
+              next
+            end
+
+            next if text.empty? && !node.name.match?(/^(img|table)$/i)
+
+            current_segment << node
+          end
+
+          segments << current_segment unless current_segment.empty?
+
+          @header_nodes = segments.shift if segments.length >= 3
+
+          segments.length >= 2 ? segments : []
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/chunker/metadata_marker_strategy.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "strategy"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      class MetadataMarkerStrategy < Strategy
+        MARKER_PATTERN = /\A\s*(?:Type:|Folder:)\s*/i
+
+        def split(doc)
+          @header_nodes = []
+          chunks = []
+          current_chunk = []
+          found_first = false
+
+          doc.children.each do |node|
+            text = node.text.strip
+            next if text.empty? && !node.name.match?(/^(img|table|hr)$/i)
+
+            if text.match?(MARKER_PATTERN)
+              found_first = true
+              chunks << current_chunk unless current_chunk.empty?
+              current_chunk = [node]
+            elsif found_first
+              current_chunk << node
+            else
+              @header_nodes << node
+            end
+          end
+
+          chunks << current_chunk unless current_chunk.empty?
+          chunks
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/chunker/numbered_question_strategy.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "strategy"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      class NumberedQuestionStrategy < Strategy
+        # Matches "1)" or "1." or "1" or "12)" etc. at start of text, but NOT single letters like "a)" because those are used for options, not question numbering
+        # We also allow for an optional "Question" prefix, e.g. "Question 1)" or "Question #: 1"
+        # NUMBERED_PATTERN = /\A\s*(\d+)\s*[.)]/
+        NUMBERED_PATTERN = /\A\s*(?:Question\s*[:#]?\s*)?(\d+)\s*[.)]/
+
+        def split(doc)
+          @header_nodes = []
+          chunks = []
+          current_chunk = []
+          found_first = false
+
+          doc.children.each do |node|
+            text = node.text.strip
+            next if text.empty? && !node.name.match?(/^(img|table|hr)$/i)
+
+            if text.match?(NUMBERED_PATTERN)
+              found_first = true
+              chunks << current_chunk unless current_chunk.empty?
+              current_chunk = [node]
+            elsif found_first
+              current_chunk << node
+            else
+              @header_nodes << node
+            end
+          end
+
+          chunks << current_chunk unless current_chunk.empty?
+          found_first ? chunks : []
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/chunker/strategy.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      class Strategy
+        attr_reader :header_nodes
+
+        def initialize
+          @header_nodes = []
+        end
+
+        # Subclasses implement this. Returns an array of chunks,
+        # where each chunk is an array of Nokogiri nodes belonging to one question.
+        # Returns empty array if this strategy doesn't apply to the document.
+        def split(doc)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/chunker.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "chunker/strategy"
+require_relative "chunker/metadata_marker_strategy"
+require_relative "chunker/numbered_question_strategy"
+require_relative "chunker/heading_split_strategy"
+require_relative "chunker/horizontal_rule_split_strategy"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Chunker
+      STRATEGIES = [
+        MetadataMarkerStrategy,
+        NumberedQuestionStrategy,
+        HeadingSplitStrategy,
+        HorizontalRuleSplitStrategy,
+      ].freeze
+
+      def self.chunk(doc)
+        warnings = []
+
+        STRATEGIES.each do |strategy_class|
+          strategy = strategy_class.new
+          chunks = strategy.split(doc)
+          next if chunks.empty?
+
+          return {
+            chunks: chunks,
+            header_nodes: strategy.header_nodes,
+            warnings: warnings,
+          }
+        end
+
+        # No strategy matched — return entire document as one chunk
+        all_nodes = doc.children.reject { |n| n.text.strip.empty? && !n.name.match?(/^(img|table|hr)$/i) }
+        warnings << "No chunking strategy matched. Treating entire document as a single question."
+
+        {
+          chunks: [all_nodes],
+          header_nodes: [],
+          warnings: warnings,
+        }
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/converter.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require "pandoc-ruby"
+require "nokogiri"
+require "securerandom"
+require "active_support/core_ext/digest/uuid"
+
+require_relative "../questions/question"
+require_relative "../questions/multiple_choice"
+require_relative "../questions/essay"
+require_relative "../questions/short_answer"
+require_relative "../questions/fill_in_the_blank"
+require_relative "../questions/matching"
+require_relative "../questions/ordering"
+require_relative "../utils"
+require_relative "chunker"
+require_relative "extractor"
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    class Converter
+      def initialize(file)
+        @file = file
+      end
+
+      def convert
+        html = normalize_to_html
+        doc = Nokogiri::HTML.fragment(html)
+        normalize_html_structure(doc)
+
+        # Chunk the document
+        chunk_result = Chunker.chunk(doc)
+        all_warnings = chunk_result[:warnings].map { |w| build_warning(w) }
+
+        if chunk_result[:chunks].length == 1
+          all_warnings << build_warning("Only 1 chunk detected — document may not be in a recognized format")
+        end
+
+        # Log header info if present
+        header_text = nil
+        unless chunk_result[:header_nodes].empty?
+          header_text = chunk_result[:header_nodes].map { |n| n.text.strip }.join(" ")
+          all_warnings << build_warning("Exam header detected: #{header_text}") unless header_text.empty?
+        end
+        activity_title = header_text&.strip&.presence || fallback_title
+
+        items = []
+        questions = []
+
+        chunk_result[:chunks].each_with_index do |chunk_nodes, index|
+          # Extract fields from this chunk
+          extraction = Extractor.extract(chunk_nodes)
+          extraction[:warnings].each do |w|
+            all_warnings << build_warning("Question #{index + 1}: #{w}", index: index, question_type: extraction[:row]["question type"])
+          end
+
+          row = extraction[:row]
+          status = extraction[:status]
+
+          # Skip completely unparseable chunks
+          if row["question text"].nil? && row["option a"].nil?
+            all_warnings << build_warning("Question #{index + 1}: Skipped — no usable content found", index: index)
+            next
+          end
+
+          next unless status == "published"
+
+          begin
+            item, question_widgets = convert_row(row, "published")
+            items << item
+            questions += question_widgets
+          rescue StandardError => e
+            title = row["title"] || "Question #{index + 1}"
+            all_warnings << build_warning("#{title}: #{e.message}", index: index, question_type: row["question type"])
+          end
+        end
+
+        {
+          activities: items.any? ? [build_activity(activity_title, items)] : [],
+          items: items,
+          questions: questions,
+          features: [],
+          errors: all_warnings,
+        }
+      end
+
+      private
+
+      def fallback_title
+        path = @file.is_a?(String) ? @file : @file.path
+        File.basename(path, ".*")
+      end
+
+      # NOTE(#2237): one activity per source file. The issue also asks that a single
+      # file containing MULTIPLE exams split into one activity each — deferred until a
+      # real classic-format sample shows what an exam boundary looks like (we have no
+      # sample defining one). Revisit when such a file exists.
+      def build_activity(title, items)
+        {
+          reference: SecureRandom.uuid,
+          title: title,
+          description: "",
+          data: {
+            config: { title: title },
+            rendering_type: "assess",
+            items: items.map { |item| { reference: item[:reference], id: item[:reference] } },
+          },
+          status: "published",
+          tags: {},
+        }
+      end
+
+      def build_warning(message, index: nil, question_type: nil)
+        {
+          error_type: "warning",
+          question_type: question_type,
+          message: message,
+          qti_item_id: nil,
+          index: index,
+        }
+      end
+
+      def normalize_html_structure(doc)
+        doc.css("p").each do |p_node|
+          br_children = p_node.css("br")
+          next if br_children.empty?
+
+          # Split the <p> at each <br> into separate <p> elements
+          segments = []
+          current_segment = []
+
+          p_node.children.each do |child|
+            if child.name == "br"
+              segments << current_segment unless current_segment.empty?
+              current_segment = []
+            else
+              current_segment << child
+            end
+          end
+          segments << current_segment unless current_segment.empty?
+
+          next if segments.length <= 1
+
+          # Replace original <p> with multiple <p> elements
+          segments.reverse_each do |segment|
+            new_p = Nokogiri::XML::Node.new("p", doc)
+            segment.each { |child| new_p.add_child(child.clone) }
+            p_node.add_next_sibling(new_p)
+          end
+          p_node.remove
+        end
+      end
+
+      def normalize_to_html
+        # Note: Pandoc Ruby takes either a file path or a string of content, but not a File object directly, so we have to handle both cases here
+        if @file.is_a?(String)
+          # File path as string
+          PandocRuby.new([@file], from: @file.split(".").last).to_html
+        elsif @file.respond_to?(:path) && @file.respond_to?(:read)
+          # File-like object (File, Tempfile, etc.)
+          source_type = @file.path.split(".").last.match(/^[a-zA-Z]+/)[0]
+          PandocRuby.new(@file.read, from: source_type).to_html
+        else
+          raise ArgumentError, "Expected a file path (String) or file-like object, got #{@file.class}"
+        end
+      end
+
+      def categories_to_tags(categories)
+        tags = {}
+        (categories || []).each do |cat|
+          parts = cat.to_s.split("/")
+          key = parts.shift&.strip
+          value = parts.join("/").strip
+          next if key.blank? || value.blank?
+
+          key = key.delete(":")[0, 255]
+          value = value[0, 255]
+          next if key.blank? || value.blank?
+
+          tags[key.to_sym] ||= []
+          tags[key.to_sym] |= [value]
+        end
+        tags
+      end
+
+      def convert_row(row, status = "published")
+        source = "<p>ExamSoft Import on #{Time.now.strftime('%Y-%m-%d')}</p>\n"
+        source += "<p>External id: #{row['question id']}</p>\n" if row["question id"].present?
+
+        question = Questions::Question.load(row)
+        # ExamSoft has a dedicated Multiple Answer question type, but Learnosity does not, so we need to update the question type and UI style for those questions
+        question_learnosity = question.to_learnosity
+        if row["question type"] == "ma"
+          question_learnosity[:data][:ui_style] = { type: "horizontal" }
+          question_learnosity[:data][:multiple_responses] = true
+        end
+
+        item = {
+          reference: SecureRandom.uuid,
+          title: row["title"] || "",
+          status: status,
+          tags: categories_to_tags(row["category"]),
+          metadata: {
+            import_date: Time.now.iso8601,
+            import_type: row["import_type"] || "examsoft",
+          },
+          source: source,
+          description: row["description"] || "",
+          questions: [
+            {
+              reference: question.reference,
+              type: question.question_type,
+            },
+          ],
+          features: [],
+          definition: {
+            widgets: [
+              {
+                reference: question.reference,
+                widget_type: "response",
+              },
+            ],
+          },
+        }
+        [item, [question_learnosity]]
+      end
+
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/extractor/correct_answer_detector.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Extractor
+      class CorrectAnswerDetector
+        ANSWER_LABEL_PATTERN = /\AAnswer:\s*(.+)/i
+
+        def initialize(nodes, options)
+          @nodes = nodes
+          @options = options
+        end
+
+        def detect
+          # First: check options for correct: true markers
+          from_options = @options.select { |opt| opt[:correct] }.map { |opt| opt[:letter] }
+          return from_options unless from_options.empty?
+
+          # Second: scan nodes for "Answer:" label
+          @nodes.each do |node|
+            text = node.text.strip
+            match = text.match(ANSWER_LABEL_PATTERN)
+            next unless match
+
+            answer_text = match[1].strip
+            # Parse comma/space-separated letters
+            letters = answer_text.split(/[\s,;]+/).map(&:strip).reject(&:empty?).map(&:downcase)
+            return letters unless letters.empty?
+          end
+
+          []
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/extractor/feedback_detector.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Extractor
+      class FeedbackDetector
+        TILDE_PATTERN = /~\s*(.+)/m
+        LABEL_PATTERN = /\A\s*(?:Explanation|Rationale):\s*(.+)/im
+        OPTION_PATTERN = /\A\s*\*?[a-oA-O]\s*[.)]/
+
+        def initialize(nodes)
+          @nodes = nodes
+        end
+
+        def detect
+          feedback_parts = []
+          collecting = false
+
+          @nodes.each do |node|
+            text = node.text.strip
+
+            if collecting
+              # Stop collecting if we hit an option line
+              break if text.match?(OPTION_PATTERN)
+              feedback_parts << text unless text.empty?
+              next
+            end
+
+            match = text.match(TILDE_PATTERN)
+            if match
+              first_part = match[1].strip
+              feedback_parts << first_part unless first_part.empty?
+              collecting = true
+            end
+          end
+
+          return feedback_parts.join(" ").gsub(/\s+/, " ").strip unless feedback_parts.empty?
+
+          @nodes.each do |node|
+            text = node.text.strip
+            match = text.match(LABEL_PATTERN)
+            return match[1].gsub(/\s+/, " ").strip if match
+          end
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/extractor/metadata_detector.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Extractor
+      class MetadataDetector
+        FOLDER_PATTERN = /Folder:\s*(.+?)(?=\s*(?:Title:|Category:|(?<=\s)\d+[.)]))/i
+        TITLE_PATTERN = /Title:\s*(.+?)(?=\s*(?:Category:|(?<=\s)\d+[.)]))/i
+        CATEGORY_PATTERN = /Category:\s*(.+?)(?=\s*\d+[.)]|\z)/i
+        TYPE_PATTERN = /Type:\s*(\S+)/i
+
+        def initialize(nodes)
+          @nodes = nodes
+        end
+
+        def detect
+          full_text = @nodes.map { |n| n.text.strip }.join(" ").gsub(/\s+/, " ")
+          result = {}
+
+          type_match = full_text.match(TYPE_PATTERN)
+          result[:type] = type_match[1].strip.downcase if type_match
+
+          folder_match = full_text.match(FOLDER_PATTERN)
+          result[:folder] = folder_match[1].strip if folder_match
+
+          title_match = full_text.match(TITLE_PATTERN)
+          result[:title] = title_match[1].strip if title_match
+
+          category_match = full_text.match(CATEGORY_PATTERN)
+          result[:categories] = category_match[1].split(/(?<!\s),(?!\s)/).map(&:strip) if category_match
+
+          result
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/extractor/options_detector.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Extractor
+      class OptionsDetector
+        OPTION_PATTERN = /\A\s*(\*?)([a-oA-O])\s*[.)]\s*(.+)/m
+
+        def initialize(nodes)
+          @nodes = nodes
+        end
+
+        def detect
+          options = []
+
+          @nodes.each do |node|
+            text = node.text.strip
+            match = text.match(OPTION_PATTERN)
+            next unless match
+
+            marker = match[1]
+            letter = match[2].downcase
+            option_text = match[3].strip
+
+            correct = marker == "*" || bold_node?(node)
+
+            options << { text: option_text, letter: letter, correct: correct }
+          end
+
+          options
+        end
+
+        private
+
+        def bold_node?(node)
+          # Check if the node's first significant child is a <strong> or <b> element
+          node.css("strong, b").any? do |bold_el|
+            bold_el.text.strip == node.text.strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/atomic_assessments_import/exam_soft/extractor/question_stem_detector.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module AtomicAssessmentsImport
+  module ExamSoft
+    module Extractor
+      class QuestionStemDetector
+        OPTION_PATTERN = /\A\s*\*?[a-oA-O]\s*[.)]/
+
+        def initialize(nodes)
+          @nodes = nodes
+        end
+
+        def detect
+          stem_node = @nodes.find do |node|
+            text = node.text.strip
+            next if text.empty?
+            next if text.match?(OPTION_PATTERN)
+
+            true
+          end
+
+          return nil unless stem_node
+
+          text = stem_node.text.strip
+
+          # Strip metadata prefixes and numbered prefix together
+          # e.g. "Folder: Geo Title: Q1 Category: Test 1) What is the capital?"
+          text = if text.match?(/\d+[.)]/m)
+                   text.sub(/\A.*?(?<!\S)\d+[.)]\s*/m, "")
+                 else
+                   # Strip standalone metadata labels if present (Folder:, Title:, Category:, Type:)
+                   text.sub(/\A\s*(?:(?:Folder|Title|Category|Type):\s*\S+\s*)*/, "")
+                 end
+
+          # Split on tilde and take the first part (remove explanation)
+          text = text.split("~").first
+
+          text = text&.gsub(/\s+/, " ")&.strip
+          text.nil? || text.empty? ? nil : text
+        end
+      end
+    end
+  end
+end