structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/xml_converter.rb

@@ -0,0 +1,651 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "shellwords"
+require "time"
+require "zlib"
+require_relative "errors"
+require_relative "format"
+require_relative "sql_text"
+require_relative "conversion_result"
+require_relative "io_support"
+require_relative "options"
+require_relative "mysql_dump_xml/table_structure"
+require_relative "mysql_dump_xml/sax_parser"
+require_relative "mysql_dump_xml/sanitizer"
+require_relative "mysql_dump_xml/table_data_filter"
+require_relative "mysql_dump_xml/invalid_character_report"
+require_relative "mysql_dump_xml/sql_emitter"
+require_relative "mysql_dump_xml/table_discovery"
+require_relative "diagnostic"
+require_relative "diagnostics_report"
+
+module StructuredDataToSql
+  SQLXML = SqlText
+
+  class XmlConverter
+    attr_reader :stats
+
+    UNESCAPED_AMP = /&(?!(?:amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);)/
+    XML_CHUNK_SIZE = 64 * 1024
+    XML_SANITIZER_TAIL = 128
+    DEFAULT_PROGRESS_INTERVAL = 5
+    ZERO_PROGRESS_WARNING_BYTES = 64 * 1024 * 1024
+
+    def initialize(
+      batch_size: 1000,
+      include_tables: nil,
+      exclude_tables: nil,
+      include_files: nil,
+      exclude_files: nil,
+      schema_only: false,
+      scrub_invalid_xml_chars: true,
+      invalid_xml_report_path: nil,
+      zero_progress_warning_bytes: ZERO_PROGRESS_WARNING_BYTES,
+      verbose: false,
+      input_gzip: false,
+      output_gzip: nil,
+      diagnostic_io: nil,
+      diagnostics_report: nil,
+      **unknown
+    )
+      if unknown.any?
+        raise ConfigurationError,
+              "Unknown XML options: #{unknown.keys.join(", ")}"
+      end
+      XmlOptions.new(batch_size:, verbose:, input_gzip:, output_gzip:)
+      unless batch_size.to_i.positive?
+        raise UsageError, "XML batch size must be greater than 0"
+      end
+
+      @batch_size = batch_size
+      @include_tables = include_tables&.to_set
+      @exclude_tables = Array(exclude_tables).to_set
+      @include_files = include_files&.to_set
+      @exclude_files = Array(exclude_files).to_set
+      @schema_only = schema_only
+      @verbose = verbose
+      @scrub_invalid_xml_chars = scrub_invalid_xml_chars
+      @invalid_xml_report_path = invalid_xml_report_path
+      @zero_progress_warning_bytes = zero_progress_warning_bytes
+      @input_gzip = input_gzip
+      @output_gzip = output_gzip
+      @diagnostic_io = diagnostic_io
+      @diagnostics_report = diagnostics_report
+      @sql_emitter = MysqlDumpXml::SqlEmitter.new
+      reset_stats
+    rescue StructuredDataToSql::ConfigurationError => e
+      raise MysqlDumpXml::ConfigurationError, e.message
+    end
+
+    def convert(
+      *arguments,
+      source: nil,
+      output: nil,
+      file_pattern: "*.xml",
+      progress_callback: nil,
+      on_progress: nil,
+      atomic: true,
+      progress_interval: DEFAULT_PROGRESS_INTERVAL,
+      input_gzip: @input_gzip,
+      output_gzip: @output_gzip
+    )
+      source ||= arguments[0]
+      output ||= arguments[1]
+      raise ConfigurationError, "source is required" if source.nil?
+      raise ConfigurationError, "output is required" if output.nil?
+      IOSupport.validate_output_target!(output)
+
+      reset_stats
+      @progress_callback = progress_callback
+      @on_progress = on_progress
+      @progress_interval = progress_interval
+      @last_progress_at = nil
+      @current_file_progress = nil
+      @current_xml_table = nil
+      @current_xml_table_included = nil
+      @conversion_started_at = Time.now
+      @total_input_bytes = 0
+      @invalid_xml_report = nil
+      @invalid_xml_report_finalized = false
+      source_label =
+        (
+          if IOSupport.readable_io?(source)
+            "(IO)"
+          else
+            Array(source).map(&:to_s).join(", ")
+          end
+        )
+      @source_path = source_label
+      @zero_progress_warning_emitted = false
+      output_path = IOSupport.writable_io?(output) ? output : Pathname(output)
+      unless IOSupport.writable_io?(output)
+        prepare_invalid_xml_report(source_label, output_path)
+      end
+      files, temporary_inputs =
+        IOSupport.discover(
+          source,
+          patterns: [file_pattern, "#{file_pattern}.gz"].uniq,
+          stream_extension: ".xml",
+          input_gzip:
+        )
+      raise UsageError, "No XML files found in #{source}" if files.empty?
+
+      files_to_process, skipped = files.partition { |file| process_file?(file) }
+      @stats[:files_skipped] += skipped.length
+      if files_to_process.empty?
+        raise UsageError,
+              "No XML files to process after filtering. All #{files.length} files were excluded."
+      end
+
+      log "\n#{"=" * 60}\nXML TO SQL CONVERTER\n#{"=" * 60}"
+      log "\nSource: #{source_label}"
+      log "Output: #{output_path}"
+      log "Files found: #{files.length}"
+      log "Files to skip: #{skipped.length}" if skipped.any?
+      log "Files to process: #{files_to_process.length}"
+      @total_input_bytes =
+        files_to_process.sum do |file|
+          begin
+            file.size
+          rescue StandardError
+            0
+          end
+        end
+      report_progress(
+        :start,
+        file_count: files_to_process.length,
+        total_input_bytes: @total_input_bytes,
+        output_path: output_path.to_s,
+        force: true
+      )
+
+      gzip_output =
+        (
+          if output_gzip.nil?
+            (!IOSupport.writable_io?(output) && output.to_s.end_with?(".gz"))
+          else
+            output_gzip
+          end
+        )
+      written_bytes =
+        IOSupport.with_output(output, gzip: gzip_output, atomic:) do |out|
+          @sql_emitter.write_header(out, source_label: source_label)
+          files_to_process.each_with_index do |file, index|
+            convert_file(
+              file,
+              out,
+              index: index + 1,
+              count: files_to_process.length
+            )
+          end
+          @sql_emitter.write_footer(out)
+        end
+      @stats[:bytes_written] = written_bytes
+      write_diagnostics_report(source_label, output_path)
+      report_progress(
+        :complete,
+        bytes_written: @stats[:bytes_written],
+        diagnostics_report: @stats[:diagnostics_report],
+        force: true
+      )
+      finalize_invalid_xml_report(success: true)
+      log "\n#{"=" * 60}\nCONVERSION COMPLETE\n#{"=" * 60}"
+      log "\nFiles processed:  #{@stats[:files_processed]}"
+      if @stats[:files_skipped].positive?
+        log "Files skipped:    #{@stats[:files_skipped]}"
+      end
+      log "Tables converted: #{@stats[:tables_processed]}"
+      if @stats[:tables_skipped].positive?
+        log "Tables skipped:   #{@stats[:tables_skipped]}"
+      end
+      log "Rows converted:   #{@stats[:rows_processed]}"
+      if @stats[:invalid_xml_chars_removed].positive?
+        log "Invalid XML chars removed: #{@stats[:invalid_xml_chars_removed]}"
+        log "Invalid XML audit: #{@invalid_xml_report.summary_path}"
+      end
+      log "Output size:      #{Format.format_size(@stats[:bytes_written])}"
+      log "\nOutput written to: #{output_path}"
+      ConversionResult.new(format: :xml, metrics: @stats)
+    rescue StructuredDataToSql::InputError => e
+      raise MysqlDumpXml::InputError, e.message
+    rescue StructuredDataToSql::OutputError => e
+      raise MysqlDumpXml::OutputError, e.message
+    rescue Nokogiri::XML::SyntaxError => e
+      raise MysqlDumpXml::ParseError, e.message
+    rescue SystemCallError, IOError, Zlib::Error => e
+      raise InputError, e.message
+    ensure
+      finalize_invalid_xml_report(success: false)
+      temporary_inputs&.each(&:unlink)
+      @progress_callback = nil
+      @on_progress = nil
+      @current_file_progress = nil
+      @current_xml_table = nil
+      @current_xml_table_included = nil
+      @invalid_xml_report = nil
+    end
+
+    def convert_file(xml_path, output, index: 1, count: 1)
+      log "\nProcessing: #{File.basename(xml_path)}"
+      file_size =
+        begin
+          xml_path.size
+        rescue StandardError
+          0
+        end
+      file_started_at = Time.now
+      rows_before = @stats[:rows_processed]
+      @current_file_progress = {
+        path: xml_path.to_s,
+        name: File.basename(xml_path),
+        index: index,
+        count: count,
+        size: file_size,
+        bytes_read: 0,
+        pass: 1,
+        passes: 1
+      }
+      report_progress(:file_start, force: true)
+      structures = {}
+      current_rows = []
+      current_table = nil
+      current_column_formats = []
+
+      parse_xml_file(xml_path) do |event_type, table, data|
+        case event_type
+        when :structure
+          set_current_xml_table(table)
+          @stats[:tables_observed] += 1
+          unless process_table?(table)
+            @stats[:tables_skipped] += 1
+            next
+          end
+
+          if current_rows.any?
+            @sql_emitter.write_rows_batch(
+              output,
+              current_table,
+              current_rows,
+              column_formats: current_column_formats
+            )
+            current_rows = []
+          end
+          structures[table] = data
+          output.write("\n-- Table: #{table}\n")
+          output.write(data.to_create_table_sql)
+          output.write("\n")
+          @stats[:tables_processed] += 1
+          report_progress(
+            :table,
+            current_table: table,
+            force: @stats[:tables_processed] == 1
+          )
+        when :row
+          @stats[:rows_observed] += 1
+          next unless process_table?(table)
+          next if @schema_only
+
+          if current_table != table
+            if current_rows.any?
+              @sql_emitter.write_rows_batch(
+                output,
+                current_table,
+                current_rows,
+                column_formats: current_column_formats
+              )
+            end
+            current_table = table
+            current_rows = []
+            current_column_formats =
+              @sql_emitter.column_formats_for(structures[table], data)
+          end
+
+          current_rows << data
+          @stats[:rows_processed] += 1
+          report_progress(
+            :rows,
+            current_table: table,
+            force: @stats[:rows_processed] == 1
+          )
+          if current_rows.length >= @batch_size
+            @sql_emitter.write_rows_batch(
+              output,
+              current_table,
+              current_rows,
+              column_formats: current_column_formats
+            )
+            current_rows = []
+          end
+        end
+      end
+
+      if current_rows.any?
+        @sql_emitter.write_rows_batch(
+          output,
+          current_table,
+          current_rows,
+          column_formats: current_column_formats
+        )
+      end
+      @stats[:files_processed] += 1
+      ledger_entry = {
+        name: File.basename(xml_path),
+        size: file_size,
+        rows: @stats[:rows_processed] - rows_before,
+        child_rows: 0,
+        elapsed: Time.now - file_started_at
+      }
+      @ledger << ledger_entry
+      report_progress(
+        :file_complete,
+        file_rows: ledger_entry[:rows],
+        file_elapsed: ledger_entry[:elapsed],
+        force: true
+      )
+      @current_file_progress = nil
+    end
+
+    private
+
+    def parse_xml_file(xml_path, &block)
+      parse_xml_file_with_nokogiri(xml_path, &block)
+    end
+
+    def parse_xml_file_with_nokogiri(xml_path, &block)
+      parser = MysqlDumpXml::SaxParser.new(&block)
+      stream_fixed_xml(xml_path) { |chunk| parser << chunk }
+      parser.finish
+    end
+
+    def stream_fixed_xml(path)
+      sanitizer =
+        MysqlDumpXml::Sanitizer.new(
+          path: path.to_s,
+          scrub_invalid_xml_chars: @scrub_invalid_xml_chars,
+          invalid_xml_handler: method(:record_invalid_xml_char)
+        )
+      table_data_filter =
+        MysqlDumpXml::TableDataFilter.new(
+          path: path.to_s,
+          table_included: method(:process_table?),
+          event_handler: method(:handle_table_data_filter_event),
+          skipped_byte_handler:
+            proc do |bytes|
+              sanitizer.skip_input_bytes(bytes)
+              track_excluded_table_data_bytes(bytes)
+            end
+        )
+      open_input(path) do |input|
+        while (chunk = input.read(XML_CHUNK_SIZE))
+          track_input_bytes(chunk.bytesize)
+          table_data_filter.feed(chunk) do |filtered|
+            sanitizer.feed(filtered) { |fixed| yield fixed }
+          end
+          warn_if_no_tables_after_bytes
+        end
+      end
+      table_data_filter.finish do |filtered|
+        sanitizer.feed(filtered) { |fixed| yield fixed }
+      end
+      sanitizer.finish { |fixed| yield fixed }
+      warn_if_no_tables_after_bytes
+    end
+
+    def handle_table_data_filter_event(event, table, included)
+      case event
+      when :table_data_start
+        set_current_xml_table(table, included: included)
+      when :table_data_end
+        set_current_xml_table(nil, included: nil)
+      end
+    end
+
+    def set_current_xml_table(table, included: process_table?(table))
+      @current_xml_table = table
+      @current_xml_table_included = table.nil? ? nil : included
+      report_progress(:bytes, force: true) if @current_file_progress && table
+    end
+
+    def track_input_bytes(bytes)
+      return unless @current_file_progress
+
+      @current_file_progress[:bytes_read] += bytes
+      @stats[:bytes_read] += bytes
+      report_progress(:bytes)
+    end
+
+    def track_excluded_table_data_bytes(bytes)
+      @stats[:excluded_table_data_bytes_skipped] += bytes
+      report_progress(:bytes)
+    end
+
+    def warn_if_no_tables_after_bytes
+      return if @zero_progress_warning_emitted
+      unless @zero_progress_warning_bytes &&
+               @zero_progress_warning_bytes.positive?
+        return
+      end
+      return if @stats[:bytes_read] < @zero_progress_warning_bytes
+      if @stats[:tables_observed].nonzero? || @stats[:rows_observed].nonzero?
+        return
+      end
+      return if @current_xml_table
+      return if @stats[:excluded_table_data_bytes_skipped].positive?
+
+      @zero_progress_warning_emitted = true
+      emit_warning(
+        "Read #{Format.format_size(@stats[:bytes_read])} from XML source #{@source_path} without discovering any tables or rows. " \
+          "Check: input path, command path, file format, current checkout. " \
+          "Recommended command: #{recommended_convert_command}"
+      )
+    end
+
+    def recommended_convert_command
+      source = @source_path ? @source_path.to_s : "SOURCE"
+      "structured-data-to-sql xml #{Shellwords.escape(source)} --output converted.sql.gz"
+    end
+
+    def emit_warning(warning)
+      diagnostic = Diagnostic.wrap(warning)
+      @stats[:warnings] << diagnostic.message
+      @stats[:diagnostics] << diagnostic
+      log "[WARN] #{diagnostic.message}"
+      report_progress(
+        :warning,
+        message: diagnostic.message,
+        diagnostic: diagnostic,
+        force: true
+      )
+    end
+
+    def write_diagnostics_report(source_label, output_path)
+      return if @diagnostics_report == false
+
+      report_path =
+        (
+          if @diagnostics_report.is_a?(String) ||
+               @diagnostics_report.is_a?(Pathname)
+            @diagnostics_report.to_s
+          else
+            DiagnosticsReport.default_path(output_path)
+          end
+        )
+      return if report_path.nil?
+
+      finished_at = Time.now
+      @stats[:diagnostics_report] = DiagnosticsReport.write(
+        report_path,
+        format: :xml,
+        source: source_label,
+        output: output_path.to_s,
+        started_at: @conversion_started_at.utc.iso8601,
+        finished_at: finished_at.utc.iso8601,
+        elapsed: finished_at - @conversion_started_at,
+        stats: @stats,
+        diagnostics: @stats[:diagnostics],
+        ledger: @ledger
+      )
+    end
+
+    def record_invalid_xml_char(file:, offset:, codepoint:)
+      @stats[:invalid_xml_chars_removed] += 1
+      @invalid_xml_report&.record(
+        file: file,
+        offset: offset,
+        codepoint: codepoint
+      )
+    end
+
+    def prepare_invalid_xml_report(source, output_path)
+      return unless @scrub_invalid_xml_chars
+
+      summary_path, events_path = invalid_xml_report_paths(output_path)
+      @invalid_xml_report =
+        MysqlDumpXml::InvalidCharacterReport.new(
+          source_path: source,
+          output_path: output_path,
+          summary_path: summary_path,
+          events_path: events_path,
+          started_at: @conversion_started_at
+        )
+    end
+
+    def finalize_invalid_xml_report(success:)
+      return unless @invalid_xml_report
+      return if @invalid_xml_report_finalized
+
+      @invalid_xml_report.finish(success: success)
+      @invalid_xml_report_finalized = true
+    end
+
+    def invalid_xml_report_paths(output_path)
+      summary_path =
+        Pathname(
+          @invalid_xml_report_path || "#{output_path}.invalid-xml-chars.json"
+        )
+      events_path =
+        if summary_path.to_s.end_with?(".summary.json")
+          Pathname(summary_path.to_s.sub(/\.summary\.json\z/, ".events.jsonl"))
+        elsif summary_path.to_s.end_with?(".json")
+          Pathname(summary_path.to_s.sub(/\.json\z/, ".events.jsonl"))
+        else
+          Pathname("#{summary_path}.events.jsonl")
+        end
+      [summary_path, events_path]
+    end
+
+    def report_progress(event, extra = {})
+      return unless @progress_callback || @on_progress
+
+      force = extra.delete(:force)
+      now = Time.now
+      if !force && @last_progress_at &&
+           now - @last_progress_at < @progress_interval
+        return
+      end
+
+      @last_progress_at = now
+      payload = progress_snapshot(event, extra)
+      @progress_callback&.call(payload)
+      @on_progress&.call(
+        ProgressEvent.new(event, payload.reject { |key, _| key == :event })
+      )
+    end
+
+    def progress_snapshot(event, extra)
+      {
+        event: event,
+        elapsed: Time.now - @conversion_started_at,
+        files_processed: @stats[:files_processed],
+        tables_processed: @stats[:tables_processed],
+        tables_observed: @stats[:tables_observed],
+        tables_skipped: @stats[:tables_skipped],
+        rows_processed: @stats[:rows_processed],
+        bytes_read: @stats[:bytes_read],
+        excluded_table_data_bytes_skipped:
+          @stats[:excluded_table_data_bytes_skipped],
+        total_input_bytes: @total_input_bytes,
+        total_work_bytes: @total_input_bytes,
+        work_bytes_read: @stats[:bytes_read],
+        warnings_count: @stats[:warnings].length,
+        current_xml_table: @current_xml_table,
+        current_xml_table_included: @current_xml_table_included,
+        current_file: @current_file_progress&.dup
+      }.merge(extra)
+    end
+
+    def temporary_output_path(path)
+      Pathname("#{path}.tmp")
+    end
+
+    def open_input(path)
+      if path.to_s.end_with?(".gz")
+        Zlib::GzipReader.open(path.to_s) { |gz| yield gz }
+      else
+        File.open(path, "rb") { |file| yield file }
+      end
+    end
+
+    def reset_stats
+      @stats = {
+        tables_processed: 0,
+        rows_processed: 0,
+        tables_observed: 0,
+        rows_observed: 0,
+        bytes_read: 0,
+        bytes_written: 0,
+        files_processed: 0,
+        files_skipped: 0,
+        tables_skipped: 0,
+        excluded_table_data_bytes_skipped: 0,
+        invalid_xml_chars_removed: 0,
+        warnings: [],
+        diagnostics: [],
+        diagnostics_report: nil
+      }
+      @ledger = []
+    end
+
+    def open_output(path, gzip: nil)
+      gzip = path.to_s.end_with?(".gz") if gzip.nil?
+      if gzip
+        Zlib::GzipWriter.open(path.to_s) { |gz| yield gz }
+      else
+        File.open(path, "w:utf-8") { |file| yield file }
+      end
+    end
+
+    def process_table?(table)
+      return false if @include_tables && !@include_tables.include?(table)
+      return false if @exclude_tables.include?(table)
+
+      true
+    end
+
+    def process_file?(file)
+      file_name = file.basename.to_s
+      base = file_name.sub(/\.gz\z/, "").sub(/\.xml\z/, "")
+      if @include_files && !@include_files.include?(file_name) &&
+           !@include_files.include?(base)
+        return false
+      end
+      if @exclude_files.include?(file_name) || @exclude_files.include?(base)
+        return false
+      end
+
+      true
+    end
+
+    def log(message)
+      @diagnostic_io.puts(message) if @verbose && @diagnostic_io
+    end
+  end
+
+  TableStructure = MysqlDumpXml::TableStructure
+  XMLTableDiscovery = MysqlDumpXml::TableDiscovery
+
+  # Compatibility aliases for the original format-specific API.
+  MysqlDumpXmlConverter = XmlConverter
+  XmlDumpConverter = XmlConverter
+end

data/lib/structured_data_to_sql/xml_dump_converter.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# Compatibility require for the original extraction filename.
+require_relative "xml_converter"

data/lib/structured_data_to_sql.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "structured_data_to_sql/version"
+require_relative "structured_data_to_sql/errors"
+require_relative "structured_data_to_sql/conversion_result"
+require_relative "structured_data_to_sql/io_support"
+require_relative "structured_data_to_sql/options"
+require_relative "structured_data_to_sql/format"
+require_relative "structured_data_to_sql/progress_reporter"
+require_relative "structured_data_to_sql/sql_text"
+require_relative "structured_data_to_sql/json_converter"
+require_relative "structured_data_to_sql/xml_converter"
+
+module StructuredDataToSql
+  FORMATS = %i[json xml].freeze
+
+  def self.convert(format:, source:, output:, **options)
+    conversion_keys = %i[
+      on_progress
+      progress_callback
+      atomic
+      progress_interval
+      file_pattern
+      input_gzip
+      output_gzip
+    ]
+    conversion_options =
+      options.select { |key, _| conversion_keys.include?(key) }
+    initializer_options =
+      options.reject { |key, _| conversion_keys.include?(key) }
+    initializer_options[:input_gzip] = options[:input_gzip] if options.key?(
+      :input_gzip
+    )
+    initializer_options[:output_gzip] = options[:output_gzip] if options.key?(
+      :output_gzip
+    )
+    converter =
+      case format.to_sym
+      when :json, :ndjson
+        initializer_options[:ndjson] = true if format.to_sym == :ndjson
+        JsonConverter
+      when :xml, :mysql_dump_xml, :mysql_xml
+        XmlConverter
+      else
+        raise ConfigurationError,
+              "Unsupported format #{format.inspect}; expected :json or :xml"
+      end
+    converter.new(**initializer_options).convert(
+      source:,
+      output:,
+      **conversion_options
+    )
+  end
+end