structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/mysql_dump_xml/table_structure.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative "../sql_text"
+
+module StructuredDataToSql
+  module MysqlDumpXml
+    class TableStructure
+      attr_reader :name, :columns, :primary_keys, :indexes, :unique_keys
+
+      def initialize(name)
+        @name = name
+        @columns = []
+        @primary_keys = []
+        @indexes = {}
+        @unique_keys = {}
+      end
+
+      def add_column(attrs)
+        col = {
+          name: attrs["Field"].to_s,
+          type: attrs["Type"] || "text",
+          null: attrs["Null"] == "YES",
+          key: attrs["Key"].to_s,
+          default: attrs["Default"],
+          extra: attrs["Extra"].to_s
+        }
+        @columns << col
+        case col[:key]
+        when "PRI"
+          @primary_keys << col[:name]
+        when "UNI"
+          @unique_keys["uk_#{col[:name]}"] = [col[:name]]
+        when "MUL"
+          @indexes["idx_#{col[:name]}"] = [col[:name]]
+        end
+      end
+
+      def column_names
+        @columns.map { |col| col[:name] }
+      end
+
+      def to_create_table_sql
+        lines =
+          @columns.map do |col|
+            definition =
+              "  #{SqlText.escape_identifier(col[:name])} #{col[:type]}"
+            definition += " NOT NULL" unless col[:null]
+            if col[:default] && col[:default] != "null"
+              default = col[:default]
+              definition +=
+                if default.start_with?("CURRENT_") || default == "NULL" ||
+                     numeric_type?(col[:type])
+                  " DEFAULT #{default}"
+                else
+                  " DEFAULT #{SqlText.escape_sql_string(default)}"
+                end
+            elsif col[:null] && col[:default] == "null"
+              definition += " DEFAULT NULL"
+            end
+            if col[:extra].downcase.include?("auto_increment")
+              definition += " AUTO_INCREMENT"
+            elsif col[:extra].downcase.include?(
+                  "on update current_timestamp"
+                ) || col[:extra].include?("DEFAULT_GENERATED")
+              definition +=
+                " ON UPDATE CURRENT_TIMESTAMP" unless definition.upcase.include?(
+                "ON UPDATE CURRENT_TIMESTAMP"
+              )
+            end
+            definition
+          end
+
+        if @primary_keys.any?
+          lines << "  PRIMARY KEY (#{@primary_keys.map { |key| SqlText.escape_identifier(key) }.join(", ")})"
+        end
+        @unique_keys.each do |name, cols|
+          lines << "  UNIQUE KEY #{SqlText.escape_identifier(name)} (#{cols.map { |col| SqlText.escape_identifier(col) }.join(", ")})"
+        end
+        @indexes.each do |name, cols|
+          lines << "  KEY #{SqlText.escape_identifier(name)} (#{cols.map { |col| SqlText.escape_identifier(col) }.join(", ")})"
+        end
+
+        "DROP TABLE IF EXISTS #{SqlText.escape_identifier(@name)};\n" \
+          "CREATE TABLE #{SqlText.escape_identifier(@name)} (\n" \
+          "#{lines.join(",\n")}\n" \
+          ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;\n"
+      end
+
+      private
+
+      def numeric_type?(type)
+        type.match?(
+          /\A(?:int|bigint|tinyint|smallint|mediumint|float|double|decimal|numeric)/i
+        )
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/mysql_dump_xml_converter.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# Compatibility require for the original format-specific filename.
+require_relative "xml_converter"

data/lib/structured_data_to_sql/options.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  class Options
+    attr_reader :values
+
+    def initialize(**values)
+      @values = defaults.merge(values).freeze
+      validate!
+      freeze
+    end
+
+    def to_h
+      @values.dup
+    end
+
+    private
+
+    def defaults
+      { batch_size: 1000, schema_only: false, verbose: false }
+    end
+
+    def validate!
+      unless @values[:batch_size].to_i.positive?
+        raise ConfigurationError, "Batch size must be greater than 0"
+      end
+    end
+  end
+
+  class JsonOptions < Options
+    private
+
+    def defaults
+      super.merge(
+        max_depth: 5,
+        ndjson: :auto,
+        graphql_unwrap: true,
+        meta_table: true,
+        recover_truncated: false,
+        input_gzip: false,
+        output_gzip: nil
+      )
+    end
+
+    def validate!
+      unless @values[:batch_size].to_i.positive?
+        raise ConfigurationError, "JSON batch size must be greater than 0"
+      end
+      unless @values[:max_depth].to_i.positive?
+        raise ConfigurationError, "JSON max depth must be greater than 0"
+      end
+      unless %i[auto json ndjson].include?(normalized_mode)
+        raise ConfigurationError,
+              "JSON input mode must be auto, json, or ndjson"
+      end
+    end
+
+    def normalized_mode
+      return :ndjson if @values[:ndjson] == true
+      return :json if @values[:ndjson] == false
+
+      @values[:ndjson].to_sym
+    end
+  end
+
+  class XmlOptions < Options
+    private
+
+    def defaults
+      super.merge(
+        scrub_invalid_xml_chars: true,
+        input_gzip: false,
+        output_gzip: nil
+      )
+    end
+
+    def validate!
+      unless @values[:batch_size].to_i.positive?
+        raise ConfigurationError, "XML batch size must be greater than 0"
+      end
+    end
+  end
+
+  Json::Options = JsonOptions
+  Xml::Options = XmlOptions
+
+  # Compatibility alias for the original format-specific API.
+  MysqlDumpXmlOptions = XmlOptions
+end

data/lib/structured_data_to_sql/progress_reporter.rb

@@ -0,0 +1,348 @@
+# frozen_string_literal: true
+
+require "io/console"
+
+require_relative "format"
+
+module StructuredDataToSql
+  # Renders converter progress events as terminal output on the diagnostic
+  # stream. It is purely a consumer of the throttled event stream the
+  # converters already emit, so it adds no per-record work: on a TTY it keeps
+  # one live status line updated in place; elsewhere it prints a plain status
+  # line at most every +min_interval+ seconds so logs show the run is alive
+  # without flooding. Finished files become fixed ledger rows, warnings are
+  # printed as they arrive, and a summary closes the run.
+  class ProgressReporter
+    TTY_INTERVAL = 0.5
+    PLAIN_INTERVAL = 30
+    ETA_WARMUP_SECONDS = 5
+    NAME_WIDTH = 28
+    PLAIN_WIDTH = 100
+    MIN_WRAP_WIDTH = 40
+    BLOCK_INDENT = "    "
+    MAX_ITEM_LINES = 3
+
+    GLYPHS = {
+      unicode: {
+        done: "✓",
+        live: "⟳",
+        warn: "!"
+      },
+      ascii: {
+        done: "[ok]",
+        live: "[..]",
+        warn: "[!!]"
+      }
+    }.freeze
+
+    def initialize(
+      io:,
+      tty: nil,
+      min_interval: nil,
+      ascii: nil,
+      profile: nil,
+      clock: nil
+    )
+      @io = io
+      @tty = tty.nil? ? (io.respond_to?(:tty?) && io.tty?) : tty
+      @min_interval = min_interval || (@tty ? TTY_INTERVAL : PLAIN_INTERVAL)
+      @glyphs =
+        GLYPHS.fetch(
+          ascii.nil? ? (@tty ? :unicode : :ascii) : (ascii ? :ascii : :unicode)
+        )
+      @profile = profile
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @live_line = nil
+      @last_status_at = nil
+      @total_input_bytes = nil
+      @last_line_blank = false
+    end
+
+    # Recommended converter throttle so the event stream matches the display
+    # cadence; the converters only build a payload when this interval passes.
+    def converter_interval
+      @min_interval
+    end
+
+    def call(event)
+      case event.type
+      when :start
+        start(event)
+      when :file_start
+        file_start(event)
+      when :pass_complete
+        @last_status_at = nil if @tty
+        status(event)
+      when :bytes, :rows, :table
+        status(event)
+      when :file_complete
+        file_complete(event)
+      when :warning
+        warning(event)
+      when :complete
+        complete(event)
+      end
+    rescue IOError, Errno::EPIPE
+      nil
+    end
+
+    private
+
+    def start(event)
+      files = event[:file_count]
+      total = event[:total_input_bytes]
+      @total_input_bytes = total
+      parts = ["Converting"]
+      parts << "#{Format.format_count(files)} file(s)" if files
+      parts << "· #{Format.format_size(total)}" if total.to_i.positive?
+      parts << "→ #{event[:output_path]}" if event[:output_path]
+      line(parts.join(" "))
+      line("Profile: #{@profile}") if @profile
+      line("")
+    end
+
+    # A new file restarts the live line immediately; plain logs keep their
+    # steady cadence across file boundaries so small files add no lines.
+    def file_start(_event)
+      @last_status_at = nil if @tty
+    end
+
+    def status(event)
+      file = event[:current_file]
+      return unless file
+
+      now = @clock.call
+      return if @last_status_at && now - @last_status_at < @min_interval
+
+      @last_status_at = now
+      text = status_text(event, file)
+      @tty ? show_live(text) : line(text)
+    end
+
+    def status_text(event, file)
+      parts = ["  #{@glyphs[:live]} #{file[:name]} #{position(file)}".rstrip]
+      parts << "pass #{file[:pass]}/#{file[:passes]}" if file[:passes].to_i > 1
+      if file[:size].to_i.positive?
+        passes = [file[:passes].to_i, 1].max
+        pass_size = file[:size] / passes
+        pass_read =
+          file[:bytes_read].to_i - ((file[:pass].to_i - 1) * pass_size)
+        pass_read = pass_read.clamp(0, pass_size)
+        parts << percent(pass_read, pass_size)
+        parts << "#{Format.format_size(pass_read)}/#{Format.format_size(pass_size)}"
+      end
+      elapsed = event[:elapsed].to_f
+      work_read = event[:work_bytes_read]
+      work_total = event[:total_work_bytes]
+      rate = Format.format_rate(work_read, elapsed) if work_read
+      parts << rate if rate
+      tail = []
+      if work_total.to_i.positive? && work_read
+        tail << "overall #{percent(work_read, work_total)}"
+      end
+      tail << "#{Format.format_duration(elapsed)} elapsed"
+      eta = eta_text(work_read, work_total, elapsed)
+      tail << eta if eta
+      "#{parts.join("  ")} · #{tail.join(" · ")}"
+    end
+
+    def eta_text(work_read, work_total, elapsed)
+      return nil unless work_total.to_i.positive? && work_read.to_i.positive?
+      return nil if elapsed < ETA_WARMUP_SECONDS
+
+      remaining = work_total - work_read
+      return nil if remaining <= 0
+
+      seconds = remaining * elapsed / work_read.to_f
+      return nil if seconds < 1
+
+      "~#{Format.format_duration(seconds)} left"
+    end
+
+    def file_complete(event)
+      file = event[:current_file] || {}
+      clear_live
+      size = file[:size].to_i / [file[:passes].to_i, 1].max
+      rows = event[:file_rows] || file[:record_count]
+      row_text = rows ? "#{Format.format_count(rows)} rows" : ""
+      row_text +=
+        " (+#{Format.format_count(event[:file_child_rows])} child)" if event[
+        :file_child_rows
+      ].to_i.positive?
+      elapsed = event[:file_elapsed]
+      line(
+        format(
+          "  %s %-#{NAME_WIDTH}s %10s %30s %8s",
+          @glyphs[:done],
+          file[:name].to_s,
+          size.positive? ? Format.format_size(size) : "",
+          row_text,
+          elapsed ? Format.format_duration(elapsed) : ""
+        ).rstrip
+      )
+    end
+
+    # A structured diagnostic renders as a block: title with headline count,
+    # wrapped summary, an abbreviated item list, detail lines, and the action.
+    # The complete lists live in the diagnostics report. Plain warnings keep
+    # the single line.
+    def warning(event)
+      clear_live
+      diagnostic = event[:diagnostic]
+      if diagnostic.nil? || diagnostic.code == :generic
+        return line("  #{@glyphs[:warn]} #{event[:message]}")
+      end
+
+      title = "  #{@glyphs[:warn]} #{diagnostic.title}"
+      title += " (#{Format.format_count(diagnostic.count)})" if diagnostic.count
+      line(title)
+      wrap(diagnostic.summary).each { |text| line(text) } if diagnostic.summary
+      item_lines(diagnostic).each { |text| line(text) }
+      diagnostic.details.each do |detail|
+        wrap(detail, first_prefix: BLOCK_INDENT).each { |text| line(text) }
+      end
+      if diagnostic.action
+        wrap(
+          diagnostic.action,
+          first_prefix: "#{BLOCK_INDENT}Action: "
+        ).each { |text| line(text) }
+      end
+      line("")
+    end
+
+    def item_lines(diagnostic)
+      return [] if diagnostic.items.empty?
+      unless diagnostic.terminal_items
+        return [
+          "#{BLOCK_INDENT}#{diagnostic.items_label || "Items"}: #{Format.format_count(diagnostic.items.length)} listed in the diagnostics report"
+        ]
+      end
+
+      label = "#{diagnostic.items_label || "Items"}: "
+      lines =
+        wrap(diagnostic.items.join(", "), first_prefix: BLOCK_INDENT + label)
+      return lines if lines.length <= MAX_ITEM_LINES
+
+      width = wrap_width
+      shown = lines.first(MAX_ITEM_LINES)
+      kept = shown.sum { |text| text.scan(/[^,\s]+(?=,|\z)/).length }
+      last = shown[-1].sub(/,?\s*\z/, "")
+      loop do
+        suffix =
+          ", … (+#{Format.format_count(diagnostic.items.length - kept)} more)"
+        break if (last + suffix).length <= width || !last.include?(", ")
+
+        last = last.sub(/,\s*[^,]*\z/, "")
+        kept -= 1
+      end
+      shown[
+        -1
+      ] = "#{last}, … (+#{Format.format_count(diagnostic.items.length - kept)} more)"
+      shown
+    end
+
+    def wrap_width
+      [(terminal_width || PLAIN_WIDTH), MIN_WRAP_WIDTH].max
+    end
+
+    def wrap(text, first_prefix: BLOCK_INDENT)
+      width = wrap_width
+      continuation = " " * first_prefix.length
+      lines = []
+      current = first_prefix.dup
+      text
+        .to_s
+        .split(/\s+/)
+        .each do |word|
+          if current.length > first_prefix.length &&
+               current.length + 1 + word.length > width
+            lines << current
+            current = continuation.dup
+          end
+          current << (current.length > continuation.length ? " " : "") << word
+        end
+      lines << current
+      lines
+    end
+
+    def complete(event)
+      clear_live
+      line("") unless @last_line_blank
+      head = ["Complete:"]
+      if event[:files_processed]
+        head << "#{Format.format_count(event[:files_processed])} file(s),"
+      end
+      rows = "#{Format.format_count(event[:rows_processed])} rows"
+      if event[:child_rows_processed].to_i.positive?
+        rows +=
+          " (+#{Format.format_count(event[:child_rows_processed])} child rows)"
+      end
+      head << rows
+      if event[:tables_processed]
+        head[-1] += ","
+        head << "#{Format.format_count(event[:tables_processed])} tables"
+      end
+      line(head.join(" "))
+
+      details = []
+      read = @total_input_bytes || event[:bytes_read]
+      details << "read #{Format.format_size(read)}" if read
+      if event[:bytes_written]
+        details << "wrote #{Format.format_size(event[:bytes_written])}"
+      end
+      details << Format.format_duration(event[:elapsed]) if event[:elapsed]
+      warnings = event[:warnings_count].to_i
+      details << "#{warnings} warning(s)#{warnings.positive? ? " (see above)" : ""}"
+      line("          #{details.join(" · ")}")
+      if event[:diagnostics_report]
+        line("Full diagnostics: #{event[:diagnostics_report]}")
+      end
+    end
+
+    def position(file)
+      return "" unless file[:index] && file[:count]
+
+      "(#{file[:index]}/#{file[:count]})"
+    end
+
+    def percent(part, whole)
+      return "" unless whole.to_i.positive?
+
+      value = [part.to_f * 100.0 / whole, 100.0].min
+      format("%d%%", value)
+    end
+
+    def show_live(text)
+      width = terminal_width
+      if width && width > 20 && text.length >= width
+        text = "#{text[0, width - 2]}…"
+      end
+      @io.write("\r#{text}\e[K")
+      @io.flush if @io.respond_to?(:flush)
+      @live_line = text
+    end
+
+    def clear_live
+      return unless @live_line
+
+      @io.write("\r\e[K")
+      @live_line = nil
+    end
+
+    def line(text)
+      clear_live
+      @io.puts(text)
+      @io.flush if @io.respond_to?(:flush)
+      @last_line_blank = text.to_s.empty?
+    end
+
+    def terminal_width
+      return nil unless @tty && @io.respond_to?(:winsize)
+
+      width = @io.winsize[1].to_i
+      width.positive? ? width : nil
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/structured_data_to_sql/sql_text.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  # MySQL/MariaDB literal and identifier escaping shared by the XML and JSON
+  # dump converters.
+  module SqlText
+    module_function
+
+    def escape_sql_string(value)
+      return "NULL" if value.nil?
+
+      escaped =
+        value
+          .to_s
+          .gsub("\\", "\\\\\\")
+          .gsub("'", "\\\\'")
+          .gsub("\n", "\\n")
+          .gsub("\r", "\\r")
+          .gsub("\t", "\\t")
+          .gsub("\x00", "\\0")
+          .gsub("\x1a", "\\Z")
+      "'#{escaped}'"
+    end
+
+    def escape_identifier(name)
+      "`#{name.to_s.gsub("`", "``")}`"
+    end
+  end
+end

data/lib/structured_data_to_sql/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  VERSION = "0.1.0"
+end