structured_data_to_sql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0ccb8d7b174afed7679305fa1e49d6892bbe7000d3816e01694aad2390773bc3
+  data.tar.gz: 522028f6c1f0935a9e58e52730c46c7682863fd3431b2ffd78bca9f74f6a6718
+SHA512:
+  metadata.gz: 9c3c997b1c817e2536deb6bdd0a86c945101b95a6d46f37cd8dbd875b3f1010f6f7538fe631edbb999fc56c105a632634fd67dd8fcccb761c3777b8eccd8ea4f
+  data.tar.gz: ca3f01ad1ac74c4720e4991f720f377c533076f1d6b54c0d4f2f0e2b3797e67e668e6fff26e249bae28a5c3b5df507b9c9b988fcda39ca8f49fe23644589f8bc

data/LICENSE

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2026 Civilized Discourse Construction Kit, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/bin/structured-data-to-sql

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "structured_data_to_sql/cli"
+
+exit StructuredDataToSql::CLI.run(ARGV)

data/lib/structured_data_to_sql/cli.rb

@@ -0,0 +1,405 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "../structured_data_to_sql"
+
+module StructuredDataToSql
+  class CLI
+    def self.run(argv, stdin: $stdin, stdout: $stdout, stderr: $stderr)
+      new(stdin:, stdout:, stderr:).run(argv)
+    end
+
+    def initialize(stdin: $stdin, stdout: $stdout, stderr: $stderr)
+      @stdin = stdin
+      @stdout = stdout
+      @stderr = stderr
+    end
+
+    def run(argv)
+      command = argv.shift
+      return print_help if command.nil? || %w[-h --help help].include?(command)
+      return print_version if %w[-v --version version].include?(command)
+
+      case command
+      when "json"
+        convert_json(argv)
+      when "xml", "mysql-xml"
+        convert_xml(argv)
+      else
+        raise UsageError,
+              "Unknown command #{command.inspect}; expected json or xml"
+      end
+      0
+    rescue Json::ParseError, MysqlDumpXml::ParseError => e
+      @stderr.puts("error: #{e.message}")
+      1
+    rescue ConfigurationError, OptionParser::ParseError => e
+      @stderr.puts("error: #{e.message}")
+      2
+    rescue Error, SystemCallError, IOError, Zlib::Error => e
+      @stderr.puts("error: #{e.message}")
+      1
+    end
+
+    private
+
+    def convert_json(argv)
+      options = common_options
+      options.merge!(max_depth: JsonConverter::DEFAULT_MAX_DEPTH, ndjson: :auto)
+      parser =
+        OptionParser.new do |opts|
+          opts.banner =
+            "Usage: structured-data-to-sql json SOURCE --output FILE [options]"
+          add_common_options(opts, options)
+          opts.on(
+            "--profile NAME",
+            "Preparation profile (khoros-api-export)"
+          ) { |v| options[:profile] = v }
+          opts.on("--records-path PATH", "Dot path containing records") do |v|
+            options[:records_path] = v
+          end
+          opts.on(
+            "--records-path-config FILE",
+            "Per-file records paths (YAML/JSON)"
+          ) { |v| options[:records_path_config] = v }
+          opts.on("--table-name NAME", "Override the root table name") do |v|
+            options[:table_name] = v
+          end
+          opts.on(
+            "--max-depth N",
+            Integer,
+            "Maximum nested object depth"
+          ) { |v| options[:max_depth] = v }
+          opts.on("--schema-dir DIR", "JSON Schema directory") do |v|
+            options[:schema_dir] = v
+          end
+          opts.on("--[no-]meta-table", "Emit JSON path metadata") do |v|
+            options[:meta_table] = v
+          end
+          opts.on("--ndjson", "Treat input as NDJSON") do
+            options[:ndjson] = true
+          end
+          opts.on("--no-ndjson", "Treat input as one JSON document") do
+            options[:ndjson] = false
+          end
+          opts.on(
+            "--json-columns LIST",
+            Array,
+            "Force paths to JSON columns"
+          ) { |v| options[:json_columns] = v }
+          opts.on("--[no-]graphql-unwrap", "Unwrap GraphQL nodes") do |v|
+            options[:graphql_unwrap] = v
+          end
+          opts.on("--raw-dates", "Do not infer DATETIME columns") do
+            options[:raw_dates] = true
+          end
+          opts.on(
+            "--recover-truncated",
+            "Recover complete records from truncated input"
+          ) { options[:recover_truncated] = true }
+        end
+      parsed = parse_source_and_output(parser, argv, options)
+      return unless parsed
+
+      source, output = parsed
+      handler = progress_handler(options)
+      JsonConverter.new(**converter_options(options)).convert(
+        source: source,
+        output: output,
+        file_pattern: options[:pattern] || "*.json",
+        atomic: true,
+        on_progress: handler,
+        **progress_interval_option(handler)
+      )
+    end
+
+    def convert_xml(argv)
+      return inspect_xml(argv.drop(1)) if argv.first == "inspect"
+
+      options = common_options
+      options[:scrub_invalid_xml_chars] = true
+      parser =
+        OptionParser.new do |opts|
+          opts.banner =
+            "Usage: structured-data-to-sql xml SOURCE --output FILE [options]"
+          add_common_options(opts, options)
+          opts.on(
+            "--[no-]scrub-invalid-xml-chars",
+            "Repair invalid XML characters"
+          ) { |v| options[:scrub_invalid_xml_chars] = v }
+          opts.on(
+            "--invalid-xml-report FILE",
+            "Write invalid-character report"
+          ) { |v| options[:invalid_xml_report_path] = v }
+        end
+      parsed = parse_source_and_output(parser, argv, options)
+      return unless parsed
+
+      source, output = parsed
+      handler = progress_handler(options)
+      XmlConverter.new(**converter_options(options)).convert(
+        source: source,
+        output: output,
+        file_pattern: options[:pattern] || "*.xml",
+        atomic: true,
+        on_progress: handler,
+        **progress_interval_option(handler)
+      )
+    end
+
+    def inspect_xml(argv)
+      options = {
+        include_files: nil,
+        exclude_files: [],
+        input_gzip: false,
+        verbose: true
+      }
+      parser =
+        OptionParser.new do |opts|
+          opts.banner =
+            "Usage: structured-data-to-sql xml inspect SOURCE [options]"
+          opts.on("--pattern GLOB", "Input filename pattern") do |value|
+            options[:pattern] = value
+          end
+          opts.on(
+            "--include-files LIST",
+            Array,
+            "Only these source files"
+          ) { |value| options[:include_files] = value }
+          opts.on(
+            "--exclude-files LIST",
+            Array,
+            "Skip these source files"
+          ) { |value| options[:exclude_files] = value }
+          opts.on("--input-gzip", "Read stdin/IO as gzip") do
+            options[:input_gzip] = true
+          end
+          opts.on("--quiet", "Suppress progress output") do
+            options[:verbose] = false
+          end
+          opts.on_tail("-h", "--help", "Show this help") do
+            options[:help] = true
+          end
+        end
+      parser.parse!(argv)
+      if options.delete(:help)
+        @stdout.puts(parser)
+        return
+      end
+      raise UsageError, parser.banner unless argv.length == 1
+
+      source_argument = argv.first
+      source = source_argument == "-" ? @stdin : source_argument
+      discovery =
+        Xml::TableDiscovery.new(
+          include_files: options[:include_files],
+          exclude_files: options[:exclude_files]
+        ).discover(
+          source,
+          file_pattern: options[:pattern] || "*.xml",
+          input_gzip: options[:input_gzip],
+          progress_callback: xml_inspection_progress_handler(options)
+        )
+      print_xml_inspection(discovery, source_argument)
+    end
+
+    def print_xml_inspection(discovery, source)
+      tables =
+        discovery.tables.sort_by do |table|
+          metadata = discovery.table_metadata[table] || {}
+          [
+            -metadata[:xml_data_bytes].to_i,
+            -Xml::TableDiscovery.total_size(metadata),
+            -metadata[:rows].to_i,
+            table
+          ]
+        end
+      table_width = ["TABLE".length, tables.map(&:length).max.to_i].max
+
+      @stdout.puts("XML table inspection")
+      @stdout.puts("Source: #{source}")
+      @stdout.puts(
+        "Files: #{discovery.files.length} selected, #{discovery.files_skipped} skipped"
+      )
+      @stdout.puts("Scanned: #{Format.format_size(discovery.bytes_scanned)}")
+      @stdout.puts("Tables: #{tables.length}")
+      @stdout.puts
+      @stdout.puts(
+        format(
+          "%-#{table_width}s %12s %12s %12s %12s %12s",
+          "TABLE",
+          "XML DATA",
+          "ROWS",
+          "DB DATA",
+          "DB INDEX",
+          "DB TOTAL"
+        )
+      )
+      tables.each do |table|
+        metadata = discovery.table_metadata[table] || {}
+        @stdout.puts(
+          format(
+            "%-#{table_width}s %12s %12s %12s %12s %12s",
+            table,
+            Format.format_size(metadata[:xml_data_bytes].to_i),
+            metadata[:rows] || "n/a",
+            optional_size(metadata[:data_length]),
+            optional_size(metadata[:index_length]),
+            database_total_size(metadata)
+          )
+        )
+      end
+      @stdout.puts
+      @stdout.puts("Convert with exclusions:")
+      @stdout.puts(
+        "  structured-data-to-sql xml SOURCE --exclude-tables TABLE1,TABLE2 --output converted.sql.gz"
+      )
+    end
+
+    def optional_size(bytes)
+      bytes.nil? ? "n/a" : Format.format_size(bytes)
+    end
+
+    def database_total_size(metadata)
+      return "n/a" unless Xml::TableDiscovery.size_metadata?(metadata)
+
+      Format.format_size(Xml::TableDiscovery.total_size(metadata))
+    end
+
+    def xml_inspection_progress_handler(options)
+      return nil unless options[:verbose]
+
+      lambda do |progress|
+        case progress[:event]
+        when :file_start
+          @stderr.puts("inspecting #{progress[:current_file][:name]}")
+        when :bytes
+          scanned = progress[:bytes_scanned].to_i
+          total = progress[:total_input_bytes].to_i
+          elapsed = progress[:elapsed].to_f
+          percent =
+            total.positive? ? format("%.1f%%", scanned * 100.0 / total) : "n/a"
+          rate = elapsed.positive? ? scanned / elapsed : 0
+          file = progress[:current_file]&.fetch(:name, nil) || "n/a"
+          @stderr.puts(
+            "inspection: #{Format.format_size(scanned)}/#{Format.format_size(total)} " \
+              "(#{percent}), #{Format.format_size(rate)}/s, #{file}, " \
+              "#{progress[:tables].to_i} table(s)"
+          )
+        when :complete
+          @stderr.puts("inspection complete: #{progress[:tables]} table(s)")
+        end
+      end
+    end
+
+    def common_options
+      {
+        batch_size: 1000,
+        include_files: nil,
+        exclude_files: [],
+        include_tables: nil,
+        exclude_tables: [],
+        schema_only: false,
+        verbose: true,
+        input_gzip: false,
+        output_gzip: nil
+      }
+    end
+
+    def add_common_options(parser, options)
+      parser.on("-o", "--output FILE", "Output .sql or .sql.gz file") do |v|
+        options[:output] = v
+      end
+      parser.on("--pattern GLOB", "Input filename pattern") do |v|
+        options[:pattern] = v
+      end
+      parser.on("--batch-size N", Integer, "Rows per INSERT") do |v|
+        options[:batch_size] = v
+      end
+      parser.on("--include-files LIST", Array, "Only these source files") do |v|
+        options[:include_files] = v
+      end
+      parser.on("--exclude-files LIST", Array, "Skip these source files") do |v|
+        options[:exclude_files] = v
+      end
+      parser.on("--include-tables LIST", Array, "Only these tables") do |v|
+        options[:include_tables] = v
+      end
+      parser.on("--exclude-tables LIST", Array, "Skip these tables") do |v|
+        options[:exclude_tables] = v
+      end
+      parser.on("--schema-only", "Emit DDL without rows") do
+        options[:schema_only] = true
+      end
+      parser.on("--quiet", "Suppress progress output") do
+        options[:verbose] = false
+      end
+      parser.on(
+        "--diagnostics-report FILE",
+        "Write the Markdown diagnostics report here (default: <output>.diagnostics.md)"
+      ) { |v| options[:diagnostics_report] = v }
+      parser.on(
+        "--no-diagnostics-report",
+        "Do not write a diagnostics report"
+      ) { options[:diagnostics_report] = false }
+      parser.on("--input-gzip", "Read stdin/IO as gzip") do
+        options[:input_gzip] = true
+      end
+      parser.on("--output-gzip", "Write stdout/IO as gzip") do
+        options[:output_gzip] = true
+      end
+      parser.on("--no-output-gzip", "Disable inferred output gzip") do
+        options[:output_gzip] = false
+      end
+      parser.on_tail("-h", "--help", "Show this help") { options[:help] = true }
+    end
+
+    def parse_source_and_output(parser, argv, options)
+      parser.parse!(argv)
+      if options.delete(:help)
+        @stdout.puts(parser)
+        return nil
+      end
+      raise UsageError, parser.banner unless argv.length == 1
+      raise UsageError, "--output is required" unless options[:output]
+
+      source = argv.first == "-" ? @stdin : argv.first
+      output = options[:output] == "-" ? @stdout : options[:output]
+      [source, output]
+    end
+
+    def converter_options(options)
+      options.reject { |key, _| %i[output pattern].include?(key) }
+    end
+
+    # Match the converter's event throttle to the display cadence: payloads
+    # are only built when the reporter can show them.
+    def progress_interval_option(handler)
+      return {} unless handler.respond_to?(:converter_interval)
+
+      { progress_interval: handler.converter_interval }
+    end
+
+    # Diagnostics go to stderr only, so SQL on stdout stays clean. The
+    # reporter consumes the converters' already-throttled event stream; the
+    # converter's own interval is matched to the display cadence so no
+    # payload is built more often than it can be shown.
+    def progress_handler(options)
+      return nil unless options[:verbose]
+
+      ProgressReporter.new(io: @stderr, profile: options[:profile])
+    end
+
+    def print_help
+      @stdout.puts(
+        "Usage: structured-data-to-sql COMMAND [options]\n\nCommands:\n  json         Convert JSON/NDJSON to MariaDB SQL\n  xml          Convert mysqldump --xml to MariaDB SQL\n  xml inspect  Inspect XML table sizes without converting"
+      )
+      0
+    end
+
+    def print_version
+      @stdout.puts("structured-data-to-sql #{VERSION}")
+      0
+    end
+  end
+end

data/lib/structured_data_to_sql/conversion_result.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  module ImmutableValue
+    private
+
+    def deep_freeze(value)
+      case value
+      when Hash
+        value.each do |key, item|
+          deep_freeze(key)
+          deep_freeze(item)
+        end
+      when Array
+        value.each { |item| deep_freeze(item) }
+      end
+      value.freeze
+    end
+  end
+
+  class ConversionResult
+    include ImmutableValue
+
+    COMMON_KEYS = %i[
+      format
+      files_processed
+      files_skipped
+      tables_processed
+      tables_skipped
+      rows_processed
+      child_rows_processed
+      bytes_read
+      bytes_written
+      warnings
+    ].freeze
+
+    def initialize(format:, metrics: {})
+      values = metrics.transform_keys(&:to_sym).merge(format: format.to_sym)
+      COMMON_KEYS.each { |key| values[key] ||= key == :warnings ? [] : 0 }
+      @values = deep_freeze(values.dup)
+      freeze
+    end
+
+    def [](key)
+      @values[key.to_sym]
+    end
+
+    def fetch(...)
+      @values.fetch(...)
+    end
+
+    def to_h
+      @values.dup
+    end
+
+    def metrics
+      @values.reject { |key, _| COMMON_KEYS.include?(key) }.freeze
+    end
+
+    COMMON_KEYS.each { |key| define_method(key) { @values[key] } }
+  end
+
+  class ProgressEvent
+    include ImmutableValue
+
+    attr_reader :type
+
+    def initialize(type, attributes = {})
+      @type = type.to_sym
+      @attributes = deep_freeze(attributes.transform_keys(&:to_sym))
+      freeze
+    end
+
+    alias event type
+
+    def [](key)
+      key.to_sym == :event ? type : @attributes[key.to_sym]
+    end
+
+    def to_h
+      @attributes.merge(event: type)
+    end
+
+    def method_missing(name, *args)
+      return @attributes[name] if args.empty? && @attributes.key?(name)
+
+      super
+    end
+
+    def respond_to_missing?(name, include_private = false)
+      @attributes.key?(name) || super
+    end
+  end
+end

data/lib/structured_data_to_sql/diagnostic.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require_relative "conversion_result"
+
+module StructuredDataToSql
+  # A structured warning: the one-line +message+ that ConversionResult#warnings
+  # has always carried, plus the parts a terminal block or a report needs —
+  # a stable +code+, a short +title+, a headline +count+, a +summary+, the
+  # complete +items+ list (field names, ids), extra +details+ lines, a
+  # suggested +action+, and +sources+ (paths to look at). Nothing here holds
+  # member values or per-row payloads.
+  class Diagnostic
+    include ImmutableValue
+
+    attr_reader :code,
+                :title,
+                :count,
+                :summary,
+                :items,
+                :items_label,
+                :terminal_items,
+                :details,
+                :action,
+                :sources,
+                :message
+
+    def initialize(
+      code:,
+      title:,
+      message:,
+      count: nil,
+      summary: nil,
+      items: [],
+      items_label: nil,
+      terminal_items: true,
+      details: [],
+      action: nil,
+      sources: []
+    )
+      @code = code.to_sym
+      @title = title.to_s
+      @message = message.to_s
+      @count = count
+      @summary = summary
+      @items = deep_freeze(Array(items).map(&:to_s))
+      @items_label = items_label
+      @terminal_items = terminal_items ? true : false
+      @details = deep_freeze(Array(details).map(&:to_s))
+      @action = action
+      @sources = deep_freeze(Array(sources).map(&:to_s))
+      freeze
+    end
+
+    def self.generic(message)
+      new(code: :generic, title: "Warning", message: message)
+    end
+
+    def self.wrap(value)
+      value.is_a?(Diagnostic) ? value : generic(value.to_s)
+    end
+
+    def to_s
+      message
+    end
+
+    def to_h
+      {
+        code: code,
+        title: title,
+        count: count,
+        summary: summary,
+        items: items,
+        items_label: items_label,
+        terminal_items: terminal_items,
+        details: details,
+        action: action,
+        sources: sources,
+        message: message
+      }
+    end
+  end
+end