canvas_link_migrator 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 993c3428360b15aea7b4eca2ee43652342f4a700e23837794ffe913e1572dea1
+  data.tar.gz: 5081c3aa602c6cf78fc3bd1e152a4c8e12f419fc6853ab72a9d09f86ba42498e
+SHA512:
+  metadata.gz: d8d3068542e30d89ddffb809c95b59ba1a2665a515a65d8f6359ec3c1f7fd204560794a3d016fc861490561215e82aad126fdb1ff0ef0f9cd8d20c6ebc12c7bb
+  data.tar.gz: b18abcf3c0b4e325cca73e10a905fea2d09c0f444230766eae955f842f8bde394765254cc9c9626ee7d6510251069e0aeed9061c81eb2c93df6b52e485aa5212

data/lib/canvas_link_migrator/imported_html_converter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2023 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "active_support/core_ext/module"
+
+module CanvasLinkMigrator
+  class ImportedHtmlConverter
+    attr_reader :link_parser, :link_resolver, :migration_id_converter
+
+    def initialize(resource_map: nil, migration_id_converter: nil)
+      @migration_id_converter = migration_id_converter || ResourceMapService.new(resource_map)
+      @link_parser = LinkParser.new(@migration_id_converter)
+      @link_resolver = LinkResolver.new(@migration_id_converter)
+    end
+
+    delegate :convert, to: :link_parser
+    delegate :resolver_links!, to: :link_resolver
+
+    def convert_exported_html(input_html)
+      new_html = link_parser.convert(input_html, "type", "lookup_id", "field")
+      replace!(new_html)
+
+      # missing links comes back as a list for all types and fields, but if the user's only
+      # sending one piece of html at a time, we only need the first set of missing links,
+      # and only the actual missing links, not the look up information we set in this method
+      bad_links = missing_links&.first&.dig(:missing_links)
+      link_parser.reset!
+      [new_html, bad_links]
+    end
+
+    def replace!(placeholder_html)
+      link_map = link_parser.unresolved_link_map
+      return unless link_map.present?
+
+      link_resolver.resolve_links!(link_map)
+      LinkReplacer.sub_placeholders!(placeholder_html, link_map.values.map(&:values).flatten)
+      placeholder_html
+    end
+
+    def missing_links
+      link_parser.unresolved_link_map.each_with_object([]) do |(item_key, field_links), bad_links|
+        field_links.each do |field, links|
+          unresolved_links = links.select { |link| link[:replaced] && (link[:missing_url] || !link[:new_value]) }
+          unresolved_links = unresolved_links.map { |link| link.slice(:link_type, :missing_url) }
+          next unless unresolved_links.any?
+
+          bad_links << { object_lookup_id: item_key[:migration_id], object_type: item_key[:type], object_field: field, missing_links: unresolved_links }
+        end
+      end
+    end
+  end
+end

data/lib/canvas_link_migrator/link_parser.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2015 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "nokogiri"
+require "digest"
+
+module CanvasLinkMigrator
+  class LinkParser
+    REFERENCE_KEYWORDS = %w[CANVAS_COURSE_REFERENCE CANVAS_OBJECT_REFERENCE WIKI_REFERENCE IMS_CC_FILEBASE IMS-CC-FILEBASE].freeze
+    LINK_PLACEHOLDER = "LINK.PLACEHOLDER"
+    KNOWN_REFERENCE_TYPES = %w[
+      announcements
+      appointment_participants
+      assignment_groups
+      assignments
+      attachments
+      calendar_events
+      context_external_tools
+      context_module_tags
+      context_modules
+      course_paces
+      created_learning_outcomes
+      discussion_entries
+      discussion_topics
+      external_feeds
+      grading_standards
+      groups
+      learning_outcome_groups
+      learning_outcome_links
+      learning_outcomes
+      linked_learning_outcomes
+      media_attachments_iframe
+      modules
+      pages
+      quizzes
+      rubrics
+      wiki
+      wiki_pages
+    ].freeze
+    CONTAINER_TYPES = %w[div p body].freeze
+    LINK_ATTRS = %w[rel href src srcset data value longdesc data-download-url].freeze
+    RCE_MEDIA_TYPES = %w[audio video].freeze
+
+    attr_reader :unresolved_link_map, :migration_query_service
+
+    def initialize(migration_query_service)
+      @migration_query_service = migration_query_service
+      reset!
+    end
+
+    def reset!
+      @unresolved_link_map = {}
+    end
+
+    def add_unresolved_link(link, item_type, mig_id, field)
+      key = { type: item_type, migration_id: mig_id }
+      @unresolved_link_map[key] ||= {}
+      @unresolved_link_map[key][field] ||= []
+      @unresolved_link_map[key][field] << link
+    end
+
+    def placeholder(old_value)
+      "#{LINK_PLACEHOLDER}_#{Digest::MD5.hexdigest(old_value)}"
+    end
+
+    def convert(html, item_type, mig_id, field, remove_outer_nodes_if_one_child: nil)
+      mig_id = mig_id.to_s
+      doc = Nokogiri::HTML5(html || "")
+
+      # Replace source tags with iframes
+      doc.search("source[data-media-id]").each do |source|
+        next unless RCE_MEDIA_TYPES.include?(source.parent.name)
+
+        media_node = source.parent
+        media_node.name = "iframe"
+        media_node["src"] = source["src"]
+        source.remove
+      end
+
+      doc.search("*").each do |node|
+        LINK_ATTRS.each do |attr|
+          convert_link(node, attr, item_type, mig_id, field)
+        end
+      end
+
+      node = doc.at_css("body")
+      return "" unless node
+
+      if remove_outer_nodes_if_one_child
+        while node.children.size == 1 && node.child.child
+          break unless CONTAINER_TYPES.member?(node.child.name) && node.child.attributes.blank?
+
+          node = node.child
+        end
+      end
+
+      node.inner_html
+    rescue Nokogiri::SyntaxError
+      ""
+    end
+
+    def convert_link(node, attr, item_type, mig_id, field)
+      return unless node[attr].present?
+
+      if attr == "value" &&
+         !(node[attr] =~ /IMS(?:-|_)CC(?:-|_)FILEBASE/ || node[attr].include?("CANVAS_COURSE_REFERENCE"))
+        return
+      end
+
+      url = node[attr].dup
+      REFERENCE_KEYWORDS.each do |ref|
+        url.gsub!("%24#{ref}%24", "$#{ref}$")
+      end
+
+      result = parse_url(url, node, attr)
+      if result[:resolved]
+        # resolved, just replace and carry on
+        new_url = result[:new_url] || url
+        unless CanvasLinkMigrator.relative_url?(new_url)
+          # perform configured substitutions
+          if (processed_url = @migration_query_service.process_domain_substitutions(new_url))
+            new_url = processed_url
+          end
+          # relative-ize absolute links outside the course but inside our domain
+          # (analogous to what is done in Api#process_incoming_html_content)
+          begin
+            uri = URI.parse(new_url)
+            account_hosts = @migration_query_service.context_hosts.map { |h| h.split(":").first }
+            if account_hosts.include?(uri.host)
+              uri.scheme = uri.host = uri.port = nil
+              new_url = uri.to_s
+            end
+          rescue URI::InvalidURIError, URI::InvalidComponentError
+            nil
+          end
+        end
+        node[attr] = new_url
+      else
+        result.delete(:resolved)
+        if result[:link_type] == :media_object
+          # because we may actually change the media comment node itself
+          # (rather than just replacing a value), we're going to
+          # replace the entire node with a placeholder
+          result[:old_value] = node.to_xml
+          result[:placeholder] = placeholder(result[:old_value])
+          placeholder_node = Nokogiri::HTML5.fragment(result[:placeholder])
+
+          node.replace(placeholder_node)
+        else
+          result[:old_value] = node[attr]
+          result[:placeholder] = placeholder(result[:old_value])
+          node[attr] = result[:placeholder]
+        end
+        add_unresolved_link(result, item_type, mig_id, field)
+      end
+    end
+
+    def unresolved(type, data = {})
+      { resolved: false, link_type: type }.merge(data)
+    end
+
+    def resolved(new_url = nil)
+      { resolved: true, new_url: new_url }
+    end
+
+    # returns a hash with resolution status and data to hold onto if unresolved
+    def parse_url(url, node, attr)
+      if url =~ /wiki_page_migration_id=(.*)/
+        unresolved(:wiki_page, migration_id: $1)
+      elsif url =~ /discussion_topic_migration_id=(.*)/
+        unresolved(:discussion_topic, migration_id: $1)
+      elsif url =~ %r{\$CANVAS_COURSE_REFERENCE\$/modules/items/([^?]*)(\?.*)?}
+        unresolved(:module_item, migration_id: $1, query: $2)
+      elsif url =~ %r{\$CANVAS_COURSE_REFERENCE\$/file_ref/([^/?#]+)(.*)}
+        unresolved(:file_ref,
+                   migration_id: $1,
+                   rest: $2,
+                   in_media_iframe: attr == "src" && ["iframe", "source"].include?(node.name) && node["data-media-id"])
+      elsif url =~ %r{(?:\$CANVAS_OBJECT_REFERENCE\$|\$WIKI_REFERENCE\$)/([^/]*)/([^?]*)(\?.*)?}
+        if KNOWN_REFERENCE_TYPES.include?($1)
+          unresolved(:object, type: $1, migration_id: $2, query: $3)
+        else
+          # If the `type` is not known, there's something amiss...
+          @migration_query_service.report_link_parse_warning($1)
+          resolved(url)
+        end
+      elsif url =~ %r{\$CANVAS_COURSE_REFERENCE\$/(.*)}
+        resolved("#{@migration_query_service.context_path}/#{$1}")
+
+      elsif url =~ %r{\$IMS(?:-|_)CC(?:-|_)FILEBASE\$/(.*)}
+        rel_path = URI::DEFAULT_PARSER.unescape($1)
+        if (attr == "href" && node["class"]&.include?("instructure_inline_media_comment")) ||
+           (attr == "src" && ["iframe", "source"].include?(node.name) && node["data-media-id"])
+          unresolved(:media_object, rel_path: rel_path)
+        else
+          unresolved(:file, rel_path: rel_path)
+        end
+      elsif (attr == "href" && node["class"]&.include?("instructure_inline_media_comment")) ||
+            (attr == "src" && ["iframe", "source"].include?(node.name) && node["data-media-id"])
+        # Course copy media reference, leave it alone
+        resolved
+      elsif @migration_query_service.supports_embedded_images && attr == "src" && (info_match = url.match(%r{\Adata:(?<mime_type>[-\w]+/[-\w+.]+)?;base64,(?<image>.*)}m))
+        result = @migration_query_service.link_embedded_image(info_match)
+        if result[:resolved]
+          resolved(result[:url])
+        else
+          unresolved(:file, rel_path: result[:url])
+        end
+      elsif # rubocop:disable Lint/DuplicateBranch
+            # Equation image, leave it alone
+            (attr == "src" && node["class"] && node["class"].include?("equation_image")) || # rubocop:disable Layout/ConditionPosition
+            # The file is in the context of an AQ, leave the link alone
+            url =~ %r{\A/assessment_questions/\d+/files/\d+} ||
+            # This points to a specific file already, leave it alone
+            url =~ %r{\A/courses/\d+/files/\d+} ||
+            !@migration_query_service.fix_relative_urls? ||
+            # It's just a link to an anchor, leave it alone
+            url.start_with?("#")
+        resolved
+      elsif CanvasLinkMigrator.relative_url?(url)
+        unresolved(:file, rel_path: URI::DEFAULT_PARSER.unescape(url))
+      else # rubocop:disable Lint/DuplicateBranch
+        resolved
+      end
+    end
+  end
+end

data/lib/canvas_link_migrator/link_replacer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2015 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+module CanvasLinkMigrator
+  class LinkReplacer
+    # returns false if no substitutions were made
+    def self.sub_placeholders!(html, links)
+      subbed = false
+      links.each do |link|
+        new_value = link[:new_value] || link[:old_value]
+        if html.gsub!(link[:placeholder], new_value)
+          link[:replaced] = true
+          subbed = true
+        end
+      end
+      subbed
+    end
+
+    def self.recursively_sub_placeholders!(object, links)
+      subbed = false
+      case object
+      when Hash
+        object.each_value { |o| subbed = true if recursively_sub_placeholders!(o, links) }
+      when Array
+        object.each { |o| subbed = true if recursively_sub_placeholders!(o, links) }
+      when String
+        subbed = sub_placeholders!(object, links)
+      end
+      subbed
+    end
+  end
+end

data/lib/canvas_link_migrator/link_resolver.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+#
+# Copyright (C) 2015 - present Instructure, Inc.
+#
+# This file is part of Canvas.
+#
+# Canvas is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, version 3 of the License.
+#
+# Canvas is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require "active_support/core_ext/object"
+
+module CanvasLinkMigrator
+  class LinkResolver
+    def initialize(migration_id_converter)
+      @migration_id_converter = migration_id_converter
+    end
+
+    def resolve_links!(link_map)
+      link_map.each_value do |field_links|
+        field_links.each_value do |links|
+          links.each do |link|
+            resolve_link!(link)
+          end
+        end
+      end
+    end
+
+    def context_path
+      @migration_id_converter.context_path
+    end
+
+    # finds the :new_value to use to replace the placeholder
+    def resolve_link!(link)
+      case link[:link_type]
+      when :wiki_page
+        if (linked_wiki_url = @migration_id_converter.convert_wiki_page_migration_id_to_slug(link[:migration_id]))
+          link[:new_value] = "#{context_path}/pages/#{linked_wiki_url}#{link[:query]}"
+        end
+      when :discussion_topic
+        if (linked_topic_id = @migration_id_converter.convert_discussion_topic_migration_id(link[:migration_id]))
+          link[:new_value] = "#{context_path}/discussion_topics/#{linked_topic_id}#{link[:query]}"
+        end
+      when :module_item
+        if (tag_id = @migration_id_converter.convert_context_module_tag_migration_id(link[:migration_id]))
+          link[:new_value] = "#{context_path}/modules/items/#{tag_id}#{link[:query]}"
+        end
+      when :object
+        type = link[:type]
+        migration_id = link[:migration_id]
+
+        type_for_url = type
+        type = "context_modules" if type == "modules"
+        type = "pages" if type == "wiki"
+        if type == "pages"
+          query = resolve_module_item_query(nil, link[:query])
+          link[:new_value] = "#{context_path}/pages/#{migration_id}#{query}"
+        elsif type == "attachments"
+          att_id = @migration_id_converter.convert_attachment_migration_id(migration_id)
+          if att_id
+            link[:new_value] = "#{context_path}/files/#{att_id}/preview"
+          end
+        elsif type == "media_attachments_iframe"
+          att_id = @migration_id_converter.convert_attachment_migration_id(migration_id)
+          link[:new_value] = att_id ? "/media_attachments_iframe/#{att_id}#{link[:query]}" : link[:old_value]
+        else
+          object_id = @migration_id_converter.convert_migration_id(type, migration_id)
+          if object_id
+            query = resolve_module_item_query(nil, link[:query])
+            link[:new_value] = "#{context_path}/#{type_for_url}/#{object_id}#{query}"
+          end
+        end
+      when :media_object
+        # because we actually might change the node itself
+        # this part is a little trickier
+        # tl;dr we've replaced the entire node with the placeholder
+        # see LinkParser for details
+        rel_path = link[:rel_path]
+        node = Nokogiri::HTML5.fragment(link[:old_value]).children.first
+        new_url = resolve_media_comment_data(node, rel_path)
+        new_url ||= resolve_relative_file_url(rel_path)
+
+        unless new_url
+          new_url ||= missing_relative_file_url(rel_path)
+          link[:missing_url] = new_url
+        end
+        if ["iframe", "source"].include?(node.name)
+          node["src"] = new_url
+        else
+          node["href"] = new_url
+        end
+        link[:new_value] = node.to_s
+      when :file
+        rel_path = link[:rel_path]
+        new_url = resolve_relative_file_url(rel_path)
+        unless new_url
+          new_url = missing_relative_file_url(rel_path)
+          link[:missing_url] = new_url
+        end
+        link[:new_value] = new_url
+      when :file_ref
+        file_id = @migration_id_converter.convert_attachment_migration_id(link[:migration_id])
+        if file_id
+          rest = link[:rest].presence || "/preview"
+
+          # Icon Maker files should not have the course
+          # context prepended to the URL. This prevents
+          # redirects to non cross-origin friendly urls
+          # during a file fetch
+          if rest.include?("icon_maker_icon=1")
+            link[:new_value] = "/files/#{file_id}#{rest}"
+          else
+            link[:new_value] = "#{context_path}/files/#{file_id}#{rest}"
+            link[:new_value] = "/media_objects_iframe?mediahref=#{link[:new_value]}" if link[:in_media_iframe]
+          end
+        end
+      else
+        raise "unrecognized link_type (#{link[:link_type]}) in unresolved link"
+      end
+    end
+
+    def resolve_module_item_query(_context, query)
+      return query unless query&.include?("module_item_id=")
+
+      original_param = query.sub("?", "").split("&").detect { |p| p.include?("module_item_id=") }
+      mig_id = original_param.split("=").last
+      tag_id = @migration_id_converter.convert_context_module_tag_migration_id(mig_id)
+      return query unless tag_id
+
+      new_param = "module_item_id=#{tag_id}"
+      query.sub(original_param, new_param)
+    end
+
+    def missing_relative_file_url(rel_path)
+      # the rel_path should already be escaped
+      File.join(URI::DEFAULT_PARSER.escape("#{context_path}/file_contents/#{@migration_id_converter.root_folder_name}"), rel_path.gsub(" ", "%20"))
+    end
+
+    def find_file_in_context(rel_path)
+      mig_id = nil
+      # This is for backward-compatibility: canvas attachment filenames are escaped
+      # with '+' for spaces and older exports have files with that instead of %20
+      alt_rel_path = rel_path.tr("+", " ")
+      if @migration_id_converter.attachment_path_id_lookup
+        mig_id ||= @migration_id_converter.attachment_path_id_lookup[rel_path]
+        mig_id ||= @migration_id_converter.attachment_path_id_lookup[alt_rel_path]
+      end
+      if !mig_id && @migration_id_converter.attachment_path_id_lookup_lower
+        mig_id ||= @migration_id_converter.attachment_path_id_lookup_lower[rel_path.downcase]
+        mig_id ||= @migration_id_converter.attachment_path_id_lookup_lower[alt_rel_path.downcase]
+      end
+
+      # This md5 comparison is here to handle faulty cartridges with the migration_id equivalent of an empty string
+      mig_id && mig_id != "gd41d8cd98f00b204e9800998ecf8427e" && @migration_id_converter.lookup_attachment_by_migration_id(mig_id)
+    end
+
+    def resolve_relative_file_url(rel_path)
+      split = rel_path.split("?")
+      qs = split.pop if split.length > 1
+      path = split.join("?")
+
+      # since we can't be sure whether a ? is part of a filename or query string, try it both ways
+      new_url = resolve_relative_file_url_with_qs(path, qs)
+      new_url ||= resolve_relative_file_url_with_qs(rel_path, "") if qs.present?
+      new_url
+    end
+
+    def resolve_relative_file_url_with_qs(rel_path, qs)
+      new_url = nil
+      rel_path_parts = Pathname.new(rel_path).each_filename.to_a
+
+      # e.g. start with "a/b/c.txt" then try "b/c.txt" then try "c.txt"
+      while new_url.nil? && !rel_path_parts.empty?
+        sub_path = File.join(rel_path_parts)
+        if (file = find_file_in_context(sub_path))
+          new_url = "#{context_path}/files/#{file.id}"
+          # support other params in the query string, that were exported from the
+          # original path components and query string. see
+          # CCHelper::file_query_string
+          params = Rack::Utils.parse_nested_query(qs.presence || "")
+          qs = []
+          new_action = ""
+          params.each do |k, v|
+            case k
+            when /canvas_qs_(.*)/
+              qs << "#{Rack::Utils.escape($1)}=#{Rack::Utils.escape(v)}"
+            when /canvas_(.*)/
+              new_action += "/#{$1}"
+            end
+          end
+          new_url += new_action.presence || "/preview"
+          new_url += "?#{qs.join("&")}" if qs.present?
+        end
+        rel_path_parts.shift
+      end
+      new_url
+    end
+
+    def media_iframe_url(media_id, media_type = nil)
+      url = "/media_objects_iframe/#{media_id}"
+      url += "?type=#{media_type}" if media_type.present?
+      url
+    end
+
+    def media_attachment_iframe_url(file_id, media_type = nil)
+      url = "/media_attachments_iframe/#{file_id}"
+      url += "?type=#{media_type}" if media_type.present?
+      url
+    end
+
+    def resolve_media_comment_data(node, rel_path)
+      if (file = find_file_in_context(rel_path[/^[^?]+/])) # strip query string for this search
+        media_id = (file.media_object&.media_id || file.media_entry_id)
+        if media_id && media_id != "maybe"
+          if ["iframe", "source"].include?(node.name)
+            node["data-media-id"] = media_id
+            if node["data-is-media-attachment"]
+              node.delete("data-is-media-attachment")
+              return media_attachment_iframe_url(file.id, node["data-media-type"])
+            else
+              return media_iframe_url(media_id, node["data-media-type"])
+            end
+          else
+            node["id"] = "media_comment_#{media_id}"
+            return "/media_objects/#{media_id}"
+          end
+        end
+      end
+
+      if node["id"] && node["id"] =~ /\Amedia_comment_(.+)\z/
+        "/media_objects/#{$1}"
+      elsif node["data-media-id"].present?
+        media_iframe_url(node["data-media-id"], node["data-media-type"])
+      else
+        node.delete("class")
+        node.delete("id")
+        node.delete("style")
+        nil
+      end
+    end
+  end
+end