structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/mysql_dump_xml/table_data_filter.rb

@@ -0,0 +1,343 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class TableDataFilter
+      XML_CHUNK_SIZE = 64 * 1024
+      CDATA_START = "<![CDATA[".b
+      CDATA_END = "]]>".b
+      COMMENT_START = "<!--".b
+      COMMENT_END = "-->".b
+      PROCESSING_START = "<?".b
+      PROCESSING_END = "?>".b
+      TABLE_DATA_START = /<\s*table_data\b/n
+      TABLE_DATA_END = %r{<\s*/\s*table_data\b}n
+      ATTR = /\b([A-Za-z_:][\w:.-]*)\s*=\s*(['"])(.*?)\2/mn
+      SCAN_TAIL_BYTES = 128
+
+      def initialize(
+        path:,
+        table_included:,
+        event_handler:,
+        skipped_byte_handler:
+      )
+        @path = path
+        @table_included = table_included
+        @event_handler = event_handler
+        @skipped_byte_handler = skipped_byte_handler
+        @buffer = +"".b
+        @passthrough_until = nil
+        @skip_until = nil
+        @skipping_table = nil
+      end
+
+      def feed(chunk, &block)
+        @buffer << chunk.to_s.b
+        drain(final: false, &block)
+      end
+
+      def finish(&block)
+        drain(final: true, &block)
+        return unless @skipping_table
+
+        raise UsageError,
+              "Missing closing </table_data> for excluded XML table #{@skipping_table.inspect} in #{@path}. " \
+                "The converter skipped that table body but could not find its end; re-export the XML or include the table to inspect the malformed block."
+      end
+
+      private
+
+      def drain(final:, &block)
+        loop do
+          progressed =
+            if @skipping_table
+              drain_excluded_table_body(final: final, &block)
+            elsif @passthrough_until
+              drain_passthrough_until(final: final, &block)
+            else
+              drain_normal(final: final, &block)
+            end
+          break unless progressed
+        end
+      end
+
+      def drain_normal(final:)
+        marker = next_normal_marker
+        unless marker
+          emit_safe_prefix(final: final) { |chunk| yield chunk }
+          return false
+        end
+
+        index, type = marker
+        if type == :table_data
+          if index.positive?
+            emit(@buffer.slice!(0, index)) { |chunk| yield chunk }
+          end
+          tag_end = start_tag_end(0)
+          unless tag_end
+            trim_oversized_partial_tag
+            return false
+          end
+
+          tag = @buffer.slice!(0, tag_end + 1)
+          table = decoded_attribute(parse_attributes(tag)["name"])
+          included = @table_included.call(table)
+          @event_handler.call(:table_data_start, table, included)
+          emit(tag) { |chunk| yield chunk }
+          if included || self_closing_tag?(tag)
+            if self_closing_tag?(tag)
+              @event_handler.call(:table_data_end, table, included)
+            end
+          else
+            @skipping_table = table
+          end
+          true
+        else
+          start, ending = skip_markers(type)
+          emit(@buffer.slice!(0, index + start.bytesize)) do |chunk|
+            yield chunk
+          end
+          @passthrough_until = ending
+          true
+        end
+      end
+
+      def drain_passthrough_until(final:)
+        index = @buffer.index(@passthrough_until)
+        if index
+          emit(
+            @buffer.slice!(0, index + @passthrough_until.bytesize)
+          ) { |chunk| yield chunk }
+          @passthrough_until = nil
+          true
+        elsif final
+          emit(@buffer.slice!(0, @buffer.bytesize)) { |chunk| yield chunk }
+          @passthrough_until = nil
+          false
+        else
+          keep = [SCAN_TAIL_BYTES, @passthrough_until.bytesize - 1].max
+          length = [@buffer.bytesize - keep, 0].max
+          if length.positive?
+            emit(@buffer.slice!(0, length)) { |chunk| yield chunk }
+          end
+          false
+        end
+      end
+
+      def drain_excluded_table_body(final:)
+        return drain_skipped_until(final: final) if @skip_until
+
+        marker = next_excluded_marker
+        unless marker
+          discard_safe_prefix(final: final)
+          return false
+        end
+
+        index, type = marker
+        if type == :table_data_end
+          if (tag_start = enclosing_tag_start(index))
+            tag_end = start_tag_end(tag_start)
+            unless tag_end
+              discard(@buffer.slice!(0, tag_start)) if tag_start.positive?
+              trim_oversized_partial_tag
+              return false
+            end
+
+            discard(@buffer.slice!(0, tag_end + 1))
+            return true
+          end
+
+          tag_end = start_tag_end(index)
+          unless tag_end
+            discard(@buffer.slice!(0, index)) if index.positive?
+            trim_oversized_partial_tag
+            return false
+          end
+
+          discard(@buffer.slice!(0, index)) if index.positive?
+          table = @skipping_table
+          emit(@buffer.slice!(0, tag_end - index + 1)) { |chunk| yield chunk }
+          @skipping_table = nil
+          @event_handler.call(:table_data_end, table, false)
+          true
+        else
+          start, ending = skip_markers(type)
+          discard(@buffer.slice!(0, index + start.bytesize))
+          @skip_until = ending
+          true
+        end
+      end
+
+      def drain_skipped_until(final:)
+        index = @buffer.index(@skip_until)
+        if index
+          discard(@buffer.slice!(0, index + @skip_until.bytesize))
+          @skip_until = nil
+          true
+        elsif final
+          discard(@buffer.slice!(0, @buffer.bytesize))
+          false
+        else
+          keep = [SCAN_TAIL_BYTES, @skip_until.bytesize - 1].max
+          length = [@buffer.bytesize - keep, 0].max
+          discard(@buffer.slice!(0, length)) if length.positive?
+          false
+        end
+      end
+
+      def next_normal_marker
+        candidates = []
+        table_data = @buffer.index(TABLE_DATA_START)
+        candidates << [table_data, :table_data] if table_data
+        cdata = @buffer.index(CDATA_START)
+        candidates << [cdata, :cdata] if cdata
+        comment = @buffer.index(COMMENT_START)
+        candidates << [comment, :comment] if comment
+        processing = @buffer.index(PROCESSING_START)
+        candidates << [processing, :processing] if processing
+        candidates.min_by(&:first)
+      end
+
+      def next_excluded_marker
+        candidates = []
+        table_data_end = @buffer.index(TABLE_DATA_END)
+        candidates << [table_data_end, :table_data_end] if table_data_end
+        cdata = @buffer.index(CDATA_START)
+        candidates << [cdata, :cdata] if cdata
+        comment = @buffer.index(COMMENT_START)
+        candidates << [comment, :comment] if comment
+        processing = @buffer.index(PROCESSING_START)
+        candidates << [processing, :processing] if processing
+        candidates.min_by(&:first)
+      end
+
+      def start_tag_end(index)
+        quote = nil
+        i = index
+        while i < @buffer.bytesize
+          char = @buffer.getbyte(i)
+          if quote
+            quote = nil if char == quote
+          elsif char == 34 || char == 39
+            quote = char
+          elsif char == 62
+            return i
+          end
+          i += 1
+        end
+        nil
+      end
+
+      def enclosing_tag_start(index)
+        tag_start = @buffer.rindex("<", index - 1)
+        return nil unless tag_start
+
+        tag_end = @buffer.rindex(">", index - 1)
+        return nil if tag_end && tag_end > tag_start
+
+        tag_start
+      end
+
+      def emit_safe_prefix(final:)
+        if final
+          unless @buffer.empty?
+            emit(@buffer.slice!(0, @buffer.bytesize)) { |chunk| yield chunk }
+          end
+        else
+          keep = SCAN_TAIL_BYTES
+          length = [@buffer.bytesize - keep, 0].max
+          if length.positive?
+            emit(@buffer.slice!(0, length)) { |chunk| yield chunk }
+          end
+        end
+      end
+
+      def discard_safe_prefix(final:)
+        keep = final ? 0 : SCAN_TAIL_BYTES
+        length = [@buffer.bytesize - keep, 0].max
+        discard(@buffer.slice!(0, length)) if length.positive?
+      end
+
+      def emit(chunk)
+        yield chunk if chunk && !chunk.empty?
+      end
+
+      def discard(chunk)
+        return if chunk.nil? || chunk.empty?
+
+        @skipped_byte_handler.call(chunk.bytesize)
+      end
+
+      def parse_attributes(tag)
+        tag
+          .scan(ATTR)
+          .each_with_object({}) do |(name, _quote, value), attrs|
+            attrs[name] = value
+          end
+      end
+
+      def decoded_attribute(value)
+        return "" if value.nil?
+
+        value =
+          value
+            .dup
+            .force_encoding("UTF-8")
+            .encode("UTF-8", invalid: :replace, undef: :replace)
+        unescape_xml_attribute(value)
+      end
+
+      def unescape_xml_attribute(value)
+        value.gsub(/&(?:amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);/) do |entity|
+          case entity
+          when "&amp;"
+            "&"
+          when "&lt;"
+            "<"
+          when "&gt;"
+            ">"
+          when "&quot;"
+            '"'
+          when "&apos;"
+            "'"
+          when /\A&#(\d+);\z/
+            Regexp.last_match(1).to_i.chr(Encoding::UTF_8)
+          when /\A&#x([0-9a-fA-F]+);\z/
+            Regexp.last_match(1).to_i(16).chr(Encoding::UTF_8)
+          else
+            entity
+          end
+        rescue RangeError
+          entity
+        end
+      end
+
+      def skip_markers(type)
+        case type
+        when :cdata
+          [CDATA_START, CDATA_END]
+        when :comment
+          [COMMENT_START, COMMENT_END]
+        when :processing
+          [PROCESSING_START, PROCESSING_END]
+        end
+      end
+
+      def self_closing_tag?(tag)
+        tag.to_s.b.sub(/>\z/n, "").rstrip.end_with?("/".b)
+      end
+
+      def trim_oversized_partial_tag
+        return if @buffer.bytesize <= XML_CHUNK_SIZE + SCAN_TAIL_BYTES
+
+        if @skipping_table
+          discard(@buffer.slice!(0, @buffer.bytesize - SCAN_TAIL_BYTES))
+        else
+          @buffer = @buffer.byteslice(1, @buffer.bytesize - 1) || +"".b
+        end
+      end
+    end
+  end
+end