structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/mysql_dump_xml/invalid_character_report.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "pathname"
+require "time"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class InvalidCharacterReport
+      attr_reader :summary_path, :events_path, :total
+
+      def initialize(
+        source_path:,
+        output_path:,
+        summary_path:,
+        events_path:,
+        started_at:
+      )
+        @source_path = source_path.to_s
+        @output_path = output_path.to_s
+        @summary_path = Pathname(summary_path)
+        @events_path = Pathname(events_path)
+        @started_at = started_at
+        @total = 0
+        @per_file = Hash.new(0)
+        @per_codepoint = Hash.new(0)
+        @events = nil
+      end
+
+      def record(file:, offset:, codepoint:)
+        open_events
+        hex = format("U+%04X", codepoint)
+        event = {
+          source_file: file.to_s,
+          input_byte_offset: offset,
+          codepoint: hex,
+          decimal: codepoint,
+          hex: hex,
+          action: "removed"
+        }
+        @events.write("#{JSON.generate(event)}\n")
+        @total += 1
+        @per_file[file.to_s] += 1
+        @per_codepoint[hex] += 1
+      end
+
+      def finish(success:)
+        return if @total.zero?
+
+        @events&.close
+        @events = nil
+        FileUtils.mkdir_p(@summary_path.dirname)
+        summary = {
+          source_path: @source_path,
+          output_path: @output_path,
+          started_at: @started_at.utc.iso8601,
+          completed_at: Time.now.utc.iso8601,
+          success: success,
+          total_scrubbed: @total,
+          per_file_totals: @per_file,
+          per_codepoint_totals: @per_codepoint,
+          events_path: @events_path.to_s
+        }
+        File.write(
+          @summary_path,
+          JSON.pretty_generate(summary),
+          mode: "w:utf-8"
+        )
+      end
+
+      private
+
+      def open_events
+        return if @events
+
+        FileUtils.mkdir_p(@events_path.dirname)
+        @events = File.open(@events_path, "w:utf-8")
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/mysql_dump_xml/sanitizer.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class Sanitizer
+      UNESCAPED_AMP = /&(?!(?:amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);)/
+      XML_SANITIZER_TAIL = 128
+      CDATA_START = "<![CDATA["
+      CDATA_END = "]]>"
+
+      def initialize(path:, scrub_invalid_xml_chars:, invalid_xml_handler:)
+        @buffer = +""
+        @in_cdata = false
+        @path = path
+        @scrub_invalid_xml_chars = scrub_invalid_xml_chars
+        @invalid_xml_handler = invalid_xml_handler
+        @input_byte_offset = 0
+      end
+
+      def feed(chunk, &block)
+        @buffer << clean_chunk(chunk)
+        drain(final: false, &block)
+      end
+
+      def finish(&block)
+        drain(final: true, &block)
+      end
+
+      def skip_input_bytes(bytes)
+        @input_byte_offset += bytes
+      end
+
+      private
+
+      def drain(final:, &block)
+        loop do
+          progressed =
+            @in_cdata ? drain_cdata(final, &block) : drain_text(final, &block)
+          break unless progressed
+        end
+      end
+
+      def drain_cdata(final)
+        index = @buffer.index(CDATA_END)
+        if index
+          yield @buffer.slice!(0, index + CDATA_END.length)
+          @in_cdata = false
+          true
+        else
+          length =
+            (
+              if final
+                @buffer.length
+              else
+                [@buffer.length - (CDATA_END.length - 1), 0].max
+              end
+            )
+          yield @buffer.slice!(0, length) if length.positive?
+          false
+        end
+      end
+
+      def drain_text(final)
+        index = @buffer.index(CDATA_START)
+        if index
+          yield fix_text(@buffer.slice!(0, index))
+          yield @buffer.slice!(0, CDATA_START.length)
+          @in_cdata = true
+          true
+        elsif final
+          unless @buffer.empty?
+            yield fix_text(@buffer.slice!(0, @buffer.length))
+          end
+          false
+        else
+          length =
+            safe_text_drain_length([@buffer.length - XML_SANITIZER_TAIL, 0].max)
+          yield fix_text(@buffer.slice!(0, length)) if length.positive?
+          false
+        end
+      end
+
+      def safe_text_drain_length(length)
+        return length unless length.positive?
+
+        amp = @buffer.rindex("&", length - 1)
+        return length unless amp
+
+        semicolon = @buffer.index(";", amp)
+        semicolon && semicolon < length ? length : amp
+      end
+
+      def fix_text(text)
+        text.gsub(UNESCAPED_AMP, "&amp;")
+      end
+
+      def clean_chunk(chunk)
+        raw = chunk.to_s.b
+        cleaned = scrub_invalid_controls(raw)
+        cleaned.force_encoding("UTF-8").encode(
+          "UTF-8",
+          invalid: :replace,
+          undef: :replace
+        )
+      end
+
+      def scrub_invalid_controls(raw)
+        result = nil
+        keep_from = 0
+        raw.bytes.each_with_index do |byte, index|
+          next unless invalid_xml_control_byte?(byte)
+
+          offset = @input_byte_offset + index
+          unless @scrub_invalid_xml_chars
+            raise UsageError,
+                  "Invalid XML control character #{format_codepoint(byte)} in #{@path} at input byte offset #{offset}. " \
+                    "XML 1.0 forbids this character; remove --no-scrub-invalid-xml-chars to scrub it and write an audit report."
+          end
+
+          result ||= +"".b
+          if index > keep_from
+            result << raw.byteslice(keep_from, index - keep_from)
+          end
+          @invalid_xml_handler&.call(
+            file: @path,
+            offset: offset,
+            codepoint: byte
+          )
+          keep_from = index + 1
+        end
+        @input_byte_offset += raw.bytesize
+        return raw unless result
+
+        if keep_from < raw.bytesize
+          result << raw.byteslice(keep_from, raw.bytesize - keep_from)
+        end
+        result
+      end
+
+      def invalid_xml_control_byte?(byte)
+        byte <= 0x08 || byte == 0x0B || byte == 0x0C ||
+          (byte >= 0x0E && byte <= 0x1F)
+      end
+
+      def format_codepoint(codepoint)
+        format("U+%04X", codepoint)
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/mysql_dump_xml/sax_parser.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "nokogiri"
+require_relative "table_structure"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class XMLStreamHandler
+      def initialize(&event_handler)
+        @event_handler = event_handler
+        @current_structure = nil
+        @current_table_data = nil
+        @current_row = nil
+        @current_field_name = nil
+        @current_field_text = nil
+      end
+
+      def tag_start(name, attrs)
+        case name
+        when "table_structure"
+          @current_structure = TableStructure.new(attrs["name"].to_s)
+        when "table_data"
+          @current_table_data = attrs["name"].to_s
+        when "row"
+          @current_row = {}
+        when "field"
+          if @current_structure
+            @current_structure.add_column(attrs)
+          elsif @current_table_data
+            @current_field_name = attrs["name"].to_s
+            @current_field_text = +""
+          end
+        end
+      end
+
+      def text(value)
+        @current_field_text << value if @current_field_name
+      end
+
+      def cdata(value)
+        @current_field_text << value if @current_field_name
+      end
+
+      def tag_end(name)
+        case name
+        when "table_structure"
+          if @current_structure
+            @event_handler.call(
+              :structure,
+              @current_structure.name,
+              @current_structure
+            )
+            @current_structure = nil
+          end
+        when "field"
+          @current_row[
+            @current_field_name
+          ] = @current_field_text if @current_field_name && @current_row
+          @current_field_name = nil
+          @current_field_text = nil
+        when "row"
+          if @current_table_data && @current_row
+            @event_handler.call(:row, @current_table_data, @current_row)
+          end
+          @current_row = nil
+        when "table_data"
+          @current_table_data = nil
+        end
+      end
+    end
+
+    class NokogiriStreamListener < Nokogiri::XML::SAX::Document
+      def initialize(&event_handler)
+        @handler = XMLStreamHandler.new(&event_handler)
+      end
+
+      def start_element(name, attrs = [])
+        @handler.tag_start(name, attrs.to_h)
+      end
+
+      def characters(value)
+        @handler.text(value)
+      end
+
+      def cdata_block(value)
+        @handler.cdata(value)
+      end
+
+      def end_element(name)
+        @handler.tag_end(name)
+      end
+    end
+
+    class SaxParser
+      def initialize(&event_handler)
+        @parser =
+          Nokogiri::XML::SAX::PushParser.new(
+            NokogiriStreamListener.new(&event_handler)
+          )
+      end
+
+      def <<(chunk)
+        @parser << chunk
+      end
+
+      def finish
+        @parser.finish
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/mysql_dump_xml/sql_emitter.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative "../sql_text"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class SqlEmitter
+      def write_header(out, source_label:)
+        out.write("-- Generated by xml-to-sql converter\n")
+        out.write("-- Source: #{source_label}\n")
+        out.write(
+          "-- Generated at: #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}\n\n"
+        )
+        out.write(
+          "-- XML dumps are generated in autocommit mode to avoid huge transaction commits.\n"
+        )
+        out.write(
+          "SET NAMES utf8mb4;\nSET sql_mode = '';\nSET FOREIGN_KEY_CHECKS = 0;\nSET UNIQUE_CHECKS = 0;\n\n"
+        )
+      end
+
+      def write_footer(out)
+        out.write("\nSET FOREIGN_KEY_CHECKS = 1;\nSET UNIQUE_CHECKS = 1;\n")
+      end
+
+      def column_formats_for(structure, sample_row)
+        if structure
+          structure.columns.map do |col|
+            {
+              name: col[:name],
+              escaped_name: SqlText.escape_identifier(col[:name]),
+              type: col[:type],
+              allows_null: col[:null],
+              unique: col[:key] == "UNI",
+              numeric: numeric_type?(col[:type])
+            }
+          end
+        else
+          sample_row.keys.map do |name|
+            {
+              name: name,
+              escaped_name: SqlText.escape_identifier(name),
+              type: "text",
+              allows_null: true,
+              unique: false,
+              numeric: false
+            }
+          end
+        end
+      end
+
+      def write_rows_batch(out, table, rows, column_formats:)
+        return if rows.empty?
+
+        buffer = +"INSERT INTO #{SqlText.escape_identifier(table)} ("
+        column_formats.each_with_index do |format, index|
+          buffer << ", " if index.positive?
+          buffer << format[:escaped_name]
+        end
+        buffer << ") VALUES\n"
+
+        rows.each_with_index do |row, row_index|
+          buffer << ",\n" if row_index.positive?
+          buffer << "  ("
+          column_formats.each_with_index do |format, column_index|
+            buffer << ", " if column_index.positive?
+            buffer << format_value(row[format[:name]], format)
+          end
+          buffer << ")"
+        end
+        buffer << ";\n"
+        out.write(buffer)
+      end
+
+      def format_value(value, format)
+        allows_null = format[:allows_null]
+        if value.nil? || value == "null"
+          return "NULL" if allows_null
+          return "0" if format[:numeric]
+
+          return "''"
+        end
+
+        if format[:numeric]
+          return allows_null ? "NULL" : "0" if value == ""
+          return value if value.to_s.match?(/\A-?\d+(?:\.\d+)?\z/)
+        end
+
+        # Repeated empty strings violate nullable unique indexes.
+        return "NULL" if format[:unique] && allows_null && value == ""
+
+        SqlText.escape_sql_string(value)
+      end
+
+      private
+
+      def numeric_type?(type)
+        type.match?(
+          /\A(?:int|bigint|tinyint|smallint|mediumint|float|double|decimal|numeric)/i
+        )
+      end
+    end
+  end
+end