structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/json_converter.rb

@@ -0,0 +1,913 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "fileutils"
+require "zlib"
+require "time"
+require "yaml"
+require_relative "errors"
+require_relative "format"
+require_relative "sql_text"
+require_relative "conversion_result"
+require_relative "io_support"
+require_relative "options"
+require_relative "json/record_streamer"
+require_relative "json/shredder"
+require_relative "json/schema_inferrer"
+require_relative "json/json_schema_loader"
+require_relative "json/sql_emitter"
+require_relative "json/profiles"
+require_relative "json/exporter_manifest"
+require_relative "diagnostic"
+require_relative "diagnostics_report"
+
+module StructuredDataToSql
+  # Converts arbitrary JSON export files into a MySQL/MariaDB SQL dump using
+  # convention-based relational shredding. Counterpart to XmlConverter:
+  # same output framing, filters, gzip handling, atomic writes, and progress
+  # callback vocabulary. Each file is streamed twice: pass 1 infers the
+  # schema (CREATE TABLEs must precede INSERTs), pass 2 emits the rows.
+  class JsonConverter
+    attr_reader :stats
+
+    DEFAULT_PROGRESS_INTERVAL = 5
+    DEFAULT_MAX_DEPTH = 5
+
+    def initialize(
+      batch_size: 1000,
+      include_tables: nil,
+      exclude_tables: nil,
+      include_files: nil,
+      exclude_files: nil,
+      schema_only: false,
+      records_path: nil,
+      records_path_config: nil,
+      table_name: nil,
+      max_depth: DEFAULT_MAX_DEPTH,
+      json_columns: nil,
+      graphql_unwrap: true,
+      raw_dates: false,
+      schema_dir: nil,
+      meta_table: true,
+      recover_truncated: false,
+      ndjson: :auto,
+      input_mode: nil,
+      verbose: false,
+      input_gzip: false,
+      output_gzip: nil,
+      profile: nil,
+      diagnostic_io: nil,
+      diagnostics_report: nil
+    )
+      ndjson = { json: false, ndjson: true, auto: :auto }.fetch(
+        input_mode.to_sym
+      ) if input_mode
+      JsonOptions.new(
+        batch_size:,
+        max_depth:,
+        ndjson:,
+        verbose:,
+        input_gzip:,
+        output_gzip:
+      )
+      unless batch_size.to_i.positive?
+        raise UsageError, "JSON batch size must be greater than 0"
+      end
+      unless max_depth.to_i.positive?
+        raise UsageError, "JSON max depth must be greater than 0"
+      end
+      if schema_dir && !Dir.exist?(schema_dir)
+        raise UsageError, "Schema directory not found: #{schema_dir}"
+      end
+      if records_path_config && !File.exist?(records_path_config)
+        raise UsageError,
+              "Records path config not found: #{records_path_config}"
+      end
+
+      @profile = normalize_profile(profile)
+      if @profile && schema_only
+        raise UsageError, "--profile cannot be used with schema-only output"
+      end
+      if @profile && (include_tables || Array(exclude_tables).any?)
+        raise UsageError,
+              "--profile cannot be combined with table include/exclude filters"
+      end
+      if @profile && (records_path || records_path_config || table_name)
+        raise UsageError,
+              "--profile cannot be combined with records-path options or a root table override"
+      end
+
+      @batch_size = batch_size
+      @include_tables = include_tables&.to_set
+      @exclude_tables = Array(exclude_tables).to_set
+      @include_files = include_files&.to_set
+      @exclude_files = Array(exclude_files).to_set
+      @schema_only = schema_only
+      @records_path = validate_records_path(records_path, "--records-path")
+      @records_path_config =
+        (
+          if records_path_config
+            load_records_path_config(records_path_config)
+          else
+            nil
+          end
+        )
+      @table_name = table_name
+      @max_depth = max_depth
+      @json_columns = json_columns
+      @graphql_unwrap = graphql_unwrap
+      @raw_dates = raw_dates
+      @schema_dir = schema_dir
+      @meta_table = meta_table
+      @recover_truncated = recover_truncated
+      @ndjson = ndjson
+      @verbose = verbose
+      @input_gzip = input_gzip
+      @output_gzip = output_gzip
+      @diagnostic_io = diagnostic_io
+      @diagnostics_report = diagnostics_report
+      @emitter = Json::SqlEmitter.new
+      reset_stats
+    rescue StructuredDataToSql::ConfigurationError => e
+      raise Json::ConfigurationError, e.message
+    end
+
+    def convert(
+      *arguments,
+      source: nil,
+      output: nil,
+      file_pattern: "*.json",
+      progress_callback: nil,
+      on_progress: nil,
+      atomic: true,
+      progress_interval: DEFAULT_PROGRESS_INTERVAL,
+      input_gzip: @input_gzip,
+      output_gzip: @output_gzip
+    )
+      source ||= arguments[0]
+      output ||= arguments[1]
+      raise ConfigurationError, "source is required" if source.nil?
+      raise ConfigurationError, "output is required" if output.nil?
+      IOSupport.validate_output_target!(output)
+
+      reset_stats
+      @progress_callback = progress_callback
+      @on_progress = on_progress
+      @progress_interval = progress_interval
+      @last_progress_at = nil
+      @current_file_progress = nil
+      @conversion_started_at = Time.now
+      @total_work_bytes = nil
+      @total_input_bytes = nil
+      @emitted_tables = {}
+      @profile_tables = {}
+      @profile_validator = profile_class::InputValidator.new if @profile
+      @pii_meta = []
+      @schema_used = false
+      @io_source = IOSupport.readable_io?(source)
+      source_label =
+        (
+          if IOSupport.readable_io?(source)
+            "(IO)"
+          else
+            Array(source).map(&:to_s).join(", ")
+          end
+        )
+      output_path = IOSupport.writable_io?(output) ? output : Pathname(output)
+      patterns = [
+        file_pattern,
+        "#{file_pattern}.gz",
+        "*.jsonl",
+        "*.jsonl.gz",
+        "*.ndjson",
+        "*.ndjson.gz"
+      ].uniq
+      files, temporary_inputs =
+        IOSupport.discover(
+          source,
+          patterns:,
+          stream_extension: ".json",
+          input_gzip:
+        )
+      raise UsageError, "No JSON files found in #{source}" if files.empty?
+
+      files_to_process, skipped = files.partition { |file| process_file?(file) }
+      @stats[:files_skipped] += skipped.length
+      if files_to_process.empty?
+        raise UsageError,
+              "No JSON files to process after filtering. All #{files.length} files were excluded."
+      end
+      validate_profile_files!(files_to_process) if @profile
+      @manifest =
+        Json::ExporterManifest.load(Json::ExporterManifest.locate(source))
+      @source_paths = quality_signal_paths(files_to_process)
+      if @profile_validator.respond_to?(:run_context=)
+        @profile_validator.run_context = {
+          manifest: @manifest,
+          source_paths: @source_paths
+        }
+      end
+
+      log "\n#{"=" * 60}\nJSON TO SQL CONVERTER\n#{"=" * 60}"
+      log "\nSource: #{source_label}"
+      log "Output: #{output_path}"
+      log "Files found: #{files.length}"
+      log "Files to skip: #{skipped.length}" if skipped.any?
+      log "Files to process: #{files_to_process.length}"
+      total_input_bytes =
+        files_to_process.sum do |file|
+          begin
+            file.size
+          rescue StandardError
+            0
+          end
+        end
+      @total_input_bytes = total_input_bytes
+      @total_work_bytes =
+        files_to_process.sum do |file|
+          IOSupport.estimated_input_bytes(file) * streaming_passes_for(file)
+        end
+      report_progress(
+        :start,
+        file_count: files_to_process.length,
+        total_input_bytes: total_input_bytes,
+        output_path: output_path.to_s,
+        force: true
+      )
+
+      gzip_output =
+        (
+          if output_gzip.nil?
+            (!IOSupport.writable_io?(output) && output.to_s.end_with?(".gz"))
+          else
+            output_gzip
+          end
+        )
+      written_bytes =
+        IOSupport.with_output(output, gzip: gzip_output, atomic:) do |out|
+          @emitter.write_header(out, source_label)
+          write_profile_preamble(out) if @profile
+          files_to_process.each_with_index do |file, index|
+            convert_file(
+              file,
+              out,
+              index: index + 1,
+              count: files_to_process.length
+            )
+          end
+          if @schema_used && @meta_table
+            @emitter.write_meta_table(out, @pii_meta)
+          end
+          write_profile(out) if @profile
+          @emitter.write_footer(out)
+        end
+      @stats[:bytes_written] = written_bytes
+      write_diagnostics_report(source_label, output_path)
+      report_progress(
+        :complete,
+        bytes_written: @stats[:bytes_written],
+        bytes_read: @stats[:bytes_read],
+        diagnostics_report: @stats[:diagnostics_report],
+        force: true
+      )
+      log "\n#{"=" * 60}\nCONVERSION COMPLETE\n#{"=" * 60}"
+      log "\nFiles processed:  #{@stats[:files_processed]}"
+      if @stats[:files_skipped].positive?
+        log "Files skipped:    #{@stats[:files_skipped]}"
+      end
+      log "Tables converted: #{@stats[:tables_processed]}"
+      if @stats[:tables_skipped].positive?
+        log "Tables skipped:   #{@stats[:tables_skipped]}"
+      end
+      log "Rows converted:   #{@stats[:rows_processed]} (+#{@stats[:child_rows_processed]} child rows)"
+      log "Output size:      #{Format.format_size(@stats[:bytes_written])}"
+      log "\nOutput written to: #{output_path}"
+      ConversionResult.new(format: :json, metrics: @stats)
+    rescue StructuredDataToSql::InputError => e
+      raise Json::InputError, e.message
+    rescue StructuredDataToSql::OutputError => e
+      raise Json::OutputError, e.message
+    rescue Oj::ParseError, EncodingError => e
+      raise Json::ParseError, e.message
+    rescue SystemCallError, IOError, Zlib::Error => e
+      raise InputError, e.message
+    ensure
+      temporary_inputs&.each(&:unlink)
+      @progress_callback = nil
+      @on_progress = nil
+      @io_source = nil
+      @current_file_progress = nil
+    end
+
+    private
+
+    def convert_file(path, out, index: 1, count: 1)
+      convert_file!(path, out, index: index, count: count)
+    rescue Oj::ParseError, EncodingError => e
+      base = File.basename(path.to_s).sub(/\.gz\z/, "").sub(/\.json\z/, "")
+      message =
+        "Malformed JSON in #{File.basename(path)} (#{e.message.sub(/ \[\S+\]\z/, "")})."
+      if truncated_json?(path)
+        message +=
+          " The file appears to be truncated — it does not end with '}' or ']'."
+        message +=
+          " Re-run with --recover-truncated to keep the complete records before the truncation point," \
+            " fix or re-export the file, or convert the others with --exclude-files #{base}."
+      else
+        message +=
+          " Fix or re-export the file, or convert the others with --exclude-files #{base}."
+      end
+      raise Json::ParseError, message
+    end
+
+    # The last non-whitespace byte of a complete JSON document is always '}'
+    # or ']' for object/array roots (the only roots these exports use).
+    def truncated_json?(path)
+      return false if path.to_s.end_with?(".gz")
+
+      tail =
+        File.open(path, "rb") do |file|
+          file.seek(-[file.size, 256].min, IO::SEEK_END)
+          file.read
+        end
+      last = tail.to_s.rstrip[-1]
+      !["}", "]"].include?(last)
+    rescue SystemCallError, IOError
+      false
+    end
+
+    def convert_file!(path, out, index: 1, count: 1)
+      log "\nProcessing: #{File.basename(path)}"
+      schema_path = find_schema(path)
+      streaming_passes = streaming_passes_for(path, schema_path)
+      file_started_at = Time.now
+      rows_before = @stats[:rows_processed]
+      child_rows_before = @stats[:child_rows_processed]
+      @current_file_progress = {
+        path: path.to_s,
+        name: File.basename(path),
+        index: index,
+        count: count,
+        size: IOSupport.estimated_input_bytes(path) * streaming_passes,
+        bytes_read: 0,
+        pass: 1,
+        passes: streaming_passes,
+        record_count: nil
+      }
+      report_progress(:file_start, force: true)
+
+      registry = Json::NameRegistry.new
+      records_path = records_path_for(path)
+      if schema_path
+        loaded =
+          Json::JsonSchemaLoader.new(
+            registry: registry,
+            max_depth: @max_depth,
+            graphql_unwrap: @graphql_unwrap,
+            raw_dates: @raw_dates
+          ).load(schema_path, records_path: records_path)
+        @schema_used = true
+        tables =
+          loaded.plans.transform_values do |plan|
+            { defs: plan.defs, child: plan.child }
+          end
+        root_table =
+          resolve_root_table({ "object_name" => loaded.root_name }, path)
+      end
+      shredder =
+        Json::Shredder.new(
+          registry: registry,
+          max_depth: @max_depth,
+          graphql_unwrap: @graphql_unwrap,
+          json_column_paths: @json_columns,
+          forced_json_paths: loaded&.forced_json
+        )
+
+      unless schema_path
+        inferrer = Json::SchemaInferrer.new(raw_dates: @raw_dates)
+        result =
+          stream_records(path) do |record, ordinal|
+            inferrer.observe(shredder.shred(record, ordinal))
+          end
+        # The emit pass re-shreds every record, so fallback/collision counts
+        # are taken from the inference pass only.
+        @stats[:json_fallback_columns] += shredder.json_fallbacks
+        @stats[:column_collisions] += registry.collisions
+        warn_total_records(path, result)
+        @current_file_progress[:record_count] = result.record_count
+        report_progress(:pass_complete, force: true)
+        tables =
+          inferrer.profiles.transform_values do |profile|
+            { defs: profile.finalize, child: profile.child? }
+          end
+        root_table = resolve_root_table(result.envelope, path)
+      end
+
+      definitions =
+        build_definitions(path, tables, root_table, strict: !schema_path.nil?)
+      definitions.each_value do |definition|
+        @emitter.write_create_table(
+          out,
+          definition[:name],
+          definition[:defs],
+          child: definition[:child]
+        )
+        @stats[:tables_processed] += 1
+        report_progress(:table, current_table: definition[:name])
+      end
+      collect_pii(loaded, definitions, schema_path) if loaded
+
+      unless @schema_only
+        @current_file_progress[:pass] = 2 unless schema_path
+        sid_counters = Hash.new(0)
+        buffers = Hash.new { |hash, rel| hash[rel] = [] }
+        emit_result =
+          stream_records(path) do |record, ordinal|
+            emit_row(
+              out,
+              shredder.shred(record, ordinal),
+              nil,
+              definitions,
+              sid_counters,
+              buffers
+            )
+          end
+        buffers.each { |rel, rows| flush_batch(out, definitions[rel], rows) }
+        warn_total_records(path, emit_result) if schema_path
+      end
+
+      @stats[:files_processed] += 1
+      ledger_entry = {
+        name: File.basename(path),
+        size: @current_file_progress[:size] / streaming_passes,
+        rows: @stats[:rows_processed] - rows_before,
+        child_rows: @stats[:child_rows_processed] - child_rows_before,
+        elapsed: Time.now - file_started_at
+      }
+      @ledger << ledger_entry
+      report_progress(
+        :file_complete,
+        file_rows: ledger_entry[:rows],
+        file_child_rows: ledger_entry[:child_rows],
+        file_elapsed: ledger_entry[:elapsed],
+        force: true
+      )
+      @current_file_progress = nil
+    end
+
+    # Streaming passes over one input: schema inference plus emit, or a
+    # single pass when a JSON Schema already describes it or only schema is
+    # requested.
+    def streaming_passes_for(path, schema_path = find_schema(path))
+      passes = schema_path ? 1 : 2
+      passes -= 1 if @schema_only
+      [passes, 1].max
+    end
+
+    def find_schema(path)
+      return nil unless @schema_dir
+
+      base = File.basename(path.to_s).sub(/\.gz\z/, "").sub(/\.json\z/, "")
+      candidate = File.join(@schema_dir, "#{base}.schema.json")
+      return Pathname(candidate) if File.exist?(candidate)
+
+      emit_warning "No JSON Schema found for #{File.basename(path)} in #{@schema_dir}; inferring schema from the data"
+      nil
+    end
+
+    def warn_total_records(path, result)
+      if result.truncated
+        @stats[:files_recovered] += 1
+        expected = result.envelope["total_records"]
+        expected_clause =
+          expected.is_a?(Integer) ? ", expected #{expected}" : ""
+        if result.skipped_lines.to_i.positive?
+          emit_warning "#{File.basename(path)} contains malformed NDJSON line(s) (#{result.parse_error}); " \
+                         "skipped #{result.skipped_lines} line(s) and recovered #{result.record_count} complete record(s)#{expected_clause}. " \
+                         "Re-export or repair the skipped line(s) for the full data."
+          return
+        end
+        emit_warning "#{File.basename(path)} is truncated (#{result.parse_error}); " \
+                       "recovered #{result.record_count} complete record(s)#{expected_clause}. " \
+                       "The remaining records are lost — re-export the file for the full data."
+        return
+      end
+
+      total_records = result.envelope["total_records"]
+      unless total_records.is_a?(Integer) &&
+               total_records != result.record_count
+        return
+      end
+
+      emit_warning "#{File.basename(path)}: total_records says #{total_records} but #{result.record_count} records were found"
+    end
+
+    # Accepts a String or a Diagnostic. +stats[:warnings]+ keeps the one-line
+    # strings; +stats[:diagnostics]+ and the :warning event carry the
+    # structured form for terminal blocks and the diagnostics report.
+    def emit_warning(warning)
+      diagnostic = Diagnostic.wrap(warning)
+      @stats[:warnings] << diagnostic.message
+      @stats[:diagnostics] << diagnostic
+      report_progress(
+        :warning,
+        message: diagnostic.message,
+        diagnostic: diagnostic,
+        force: true
+      )
+      log "[WARN] #{diagnostic.message}"
+    end
+
+    # Paths of the exporter's quality-signal files among the inputs, handed
+    # to the profile so diagnostics can point at them.
+    def quality_signal_paths(files)
+      %w[errors gaps].to_h do |table|
+        match = files.find { |file| json_basename(file) == table }
+        [table, match&.to_s]
+      end
+    end
+
+    def write_diagnostics_report(source_label, output_path)
+      return if @diagnostics_report == false
+
+      report_path =
+        (
+          if @diagnostics_report.is_a?(String) ||
+               @diagnostics_report.is_a?(Pathname)
+            @diagnostics_report.to_s
+          else
+            DiagnosticsReport.default_path(output_path)
+          end
+        )
+      return if report_path.nil?
+
+      finished_at = Time.now
+      @stats[:diagnostics_report] = DiagnosticsReport.write(
+        report_path,
+        format: :json,
+        source: source_label,
+        output: output_path.to_s,
+        profile: @profile,
+        started_at: @conversion_started_at.utc.iso8601,
+        finished_at: finished_at.utc.iso8601,
+        elapsed: finished_at - @conversion_started_at,
+        stats: @stats,
+        input_bytes: @total_input_bytes,
+        diagnostics: @stats[:diagnostics],
+        ledger: @ledger,
+        manifest: @manifest,
+        source_paths: @source_paths
+      )
+    end
+
+    def collect_pii(loaded, definitions, schema_path)
+      loaded.pii.each do |entry|
+        definition = definitions[entry.table_rel]
+        next unless definition
+
+        @pii_meta << [
+          definition[:name],
+          entry.column,
+          entry.json_path,
+          1,
+          entry.note,
+          File.basename(schema_path)
+        ]
+      end
+    end
+
+    def build_definitions(path, tables, root_table, strict: false)
+      definitions = {}
+      tables.each do |rel, table|
+        name = final_table_name(root_table, rel)
+        if (previous = @emitted_tables[name])
+          raise UsageError,
+                "Table name '#{name}' is produced by both #{previous} and #{path}. Rename one file or use --include-files/--exclude-files."
+        end
+
+        unless process_table?(name, root: root_table)
+          @stats[:tables_skipped] += 1
+          next
+        end
+
+        @emitted_tables[name] = path.to_s
+        meta =
+          (
+            if table[:child]
+              Json::SqlEmitter::META_CHILD
+            else
+              Json::SqlEmitter::META_ROOT
+            end
+          )
+        column_names = table[:defs].map(&:name)
+        definitions[rel] = {
+          name: name,
+          defs: table[:defs],
+          child: table[:child],
+          columns: meta + column_names,
+          known_columns: strict ? column_names.to_set : nil,
+          context_columns: column_names.grep(/\Acontext_/).sort
+        }
+        @profile_tables[name] = meta + column_names if @profile
+      end
+      definitions
+    end
+
+    def emit_row(out, row, parent_sid, definitions, sid_counters, buffers)
+      sid = (sid_counters[row.table_rel] += 1)
+      definition = definitions[row.table_rel]
+      if definition
+        if @profile_validator
+          @profile_validator.observe(
+            definition[:name],
+            row.columns,
+            sid,
+            parent_sid: definition[:child] ? parent_sid : nil,
+            context_columns: definition[:context_columns]
+          )
+        end
+        values =
+          if definition[:child]
+            [
+              sid.to_s,
+              parent_sid.to_s,
+              SqlText.escape_sql_string(row.parent_natural_id),
+              row.ordinal.to_s
+            ]
+          else
+            [sid.to_s]
+          end
+        values.concat(
+          definition[:defs].map do |col|
+            @emitter.format_value(row.columns[col.name], col)
+          end
+        )
+        if (known = definition[:known_columns])
+          row.columns.each_key do |column|
+            @stats[:dropped_values] += 1 unless known.include?(column)
+          end
+        end
+        buffer = buffers[row.table_rel]
+        buffer << values
+        if definition[:child]
+          @stats[:child_rows_processed] += 1
+        else
+          @stats[:rows_processed] += 1
+        end
+        report_progress(:rows)
+        flush_batch(out, definition, buffer) if buffer.length >= @batch_size
+      end
+      row.children.each do |child|
+        emit_row(out, child, sid, definitions, sid_counters, buffers)
+      end
+    end
+
+    def flush_batch(out, definition, rows)
+      return if definition.nil? || rows.empty?
+
+      @emitter.write_batch(out, definition[:name], definition[:columns], rows)
+      rows.clear
+    end
+
+    def resolve_root_table(envelope, path)
+      base = @table_name
+      base ||= envelope["object_name"] if envelope["object_name"].is_a?(String)
+      base ||= "data" if @io_source
+      base ||= json_basename(path)
+      Json::NameRegistry.sanitize(base)
+    end
+
+    def final_table_name(root_table, rel)
+      if rel.empty?
+        root_table
+      else
+        Json::NameRegistry.truncate("#{root_table}_#{rel}")
+      end
+    end
+
+    def stream_records(path, &block)
+      open_input(path) do |io|
+        counting = Json::CountingIO.new(io) { |bytes| track_input_bytes(bytes) }
+        Json::RecordStreamer.new(
+          records_path: records_path_for(path)
+        ).each_record(
+          counting,
+          recover: @recover_truncated,
+          ndjson: @ndjson,
+          &block
+        )
+      end
+    end
+
+    # Returns the records path to use for +path+. Per-file config takes
+    # precedence over the global --records-path flag.
+    def records_path_for(path)
+      if @records_path_config
+        base = json_basename(path)
+        return @records_path_config[base] if @records_path_config.key?(base)
+      end
+      @records_path
+    end
+
+    # Loads the YAML (or JSON) records-path config file and returns a Hash
+    # mapping filename base (without extension) to a dot-notation path string.
+    def load_records_path_config(config_path)
+      raw =
+        YAML.safe_load(
+          File.read(config_path),
+          permitted_classes: [],
+          permitted_symbols: [],
+          aliases: true
+        )
+      unless raw.is_a?(Hash)
+        raise UsageError,
+              "Records path config must be a YAML mapping of filename => path (got #{raw.class})"
+      end
+
+      raw.each_with_object({}) do |(key, value), config|
+        unless value.is_a?(String)
+          raise UsageError,
+                "Records path config values must be strings, got #{value.inspect}"
+        end
+
+        config[key.to_s] = validate_records_path(
+          value,
+          "records path config entry #{key.inspect}"
+        )
+      end
+    rescue Psych::Exception => e
+      raise UsageError,
+            "Could not parse records path config #{config_path}: #{e.message}"
+    end
+
+    def json_basename(path)
+      File
+        .basename(path.to_s)
+        .sub(/\.gz\z/, "")
+        .sub(/\.(?:json|jsonl|ndjson)\z/, "")
+    end
+
+    def validate_records_path(path, label)
+      return nil if path.nil?
+
+      value = path.to_s.strip
+      if value.empty? || value.start_with?(".") || value.end_with?(".") ||
+           value.split(".", -1).any?(&:empty?)
+        raise UsageError,
+              "Invalid #{label}: #{path.inspect}. Use dot-notation with non-empty path segments."
+      end
+      value
+    end
+
+    def normalize_profile(profile)
+      return nil if profile.nil?
+
+      normalized = profile.to_s.downcase.tr("-", "_").to_sym
+      return normalized if Json::Profiles::REGISTRY.key?(normalized)
+
+      expected =
+        Json::Profiles::REGISTRY.keys.map { |name| name.to_s.tr("_", "-") }
+      raise UsageError,
+            "Unknown JSON preparation profile #{profile.inspect}; expected #{expected.join(", ")}"
+    end
+
+    def profile_class
+      Json::Profiles::REGISTRY.fetch(@profile)
+    end
+
+    def write_profile(out)
+      @profile_validator.validate!
+      warnings =
+        if @profile_validator.respond_to?(:diagnostics)
+          @profile_validator.diagnostics
+        else
+          @profile_validator.warnings
+        end
+      warnings.each { |warning| emit_warning(warning) }
+      profile_class.new(@profile_tables).write(out)
+    end
+
+    def write_profile_preamble(out)
+      profile_class.write_preamble(out)
+    end
+
+    def validate_profile_files!(files)
+      available = files.map { |file| json_basename(file) }
+      missing = profile_class::REQUIRED_SCHEMA.keys - available
+      return if missing.empty?
+
+      raise UsageError,
+            "Khoros API profile required collections are missing or filtered: #{missing.join(", ")}"
+    end
+
+    def track_input_bytes(bytes)
+      return unless @current_file_progress
+
+      @current_file_progress[:bytes_read] += bytes
+      @stats[:bytes_read] += bytes
+      report_progress(:bytes)
+    end
+
+    def report_progress(event, extra = {})
+      return unless @progress_callback || @on_progress
+
+      force = extra.delete(:force)
+      now = Time.now
+      if !force && @last_progress_at &&
+           now - @last_progress_at < @progress_interval
+        return
+      end
+
+      @last_progress_at = now
+      payload = {
+        event: event,
+        elapsed: Time.now - @conversion_started_at,
+        files_processed: @stats[:files_processed],
+        tables_processed: @stats[:tables_processed],
+        rows_processed: @stats[:rows_processed],
+        child_rows_processed: @stats[:child_rows_processed],
+        work_bytes_read: @stats[:bytes_read],
+        total_work_bytes: @total_work_bytes,
+        warnings_count: @stats[:warnings].length,
+        current_file: @current_file_progress&.dup
+      }.merge(extra)
+      @progress_callback&.call(payload)
+      @on_progress&.call(
+        ProgressEvent.new(event, payload.reject { |key, _| key == :event })
+      )
+    end
+
+    def temporary_output_path(path)
+      Pathname("#{path}.tmp")
+    end
+
+    def open_input(path)
+      if path.to_s.end_with?(".gz")
+        Zlib::GzipReader.open(path.to_s) { |gz| yield gz }
+      else
+        File.open(path, "rb") { |file| yield file }
+      end
+    end
+
+    def open_output(path, gzip: nil)
+      gzip = path.to_s.end_with?(".gz") if gzip.nil?
+      if gzip
+        Zlib::GzipWriter.open(path.to_s) { |gz| yield gz }
+      else
+        File.open(path, "w:utf-8") { |file| yield file }
+      end
+    end
+
+    def reset_stats
+      @stats = {
+        tables_processed: 0,
+        rows_processed: 0,
+        child_rows_processed: 0,
+        bytes_read: 0,
+        bytes_written: 0,
+        files_processed: 0,
+        files_skipped: 0,
+        tables_skipped: 0,
+        column_collisions: 0,
+        json_fallback_columns: 0,
+        dropped_values: 0,
+        files_recovered: 0,
+        warnings: [],
+        diagnostics: [],
+        diagnostics_report: nil
+      }
+      @ledger = []
+      @manifest = nil
+      @source_paths = {}
+    end
+
+    def process_table?(table, root:)
+      if @exclude_tables.include?(root) || @exclude_tables.include?(table)
+        return false
+      end
+      if @include_tables
+        return @include_tables.include?(table) || @include_tables.include?(root)
+      end
+
+      true
+    end
+
+    def process_file?(file)
+      file_name = file.basename.to_s
+      base = file_name.sub(/\.gz\z/, "").sub(/\.(?:json|jsonl|ndjson)\z/, "")
+      if @include_files && !@include_files.include?(file_name) &&
+           !@include_files.include?(base)
+        return false
+      end
+      if @exclude_files.include?(file_name) || @exclude_files.include?(base)
+        return false
+      end
+
+      true
+    end
+
+    def log(message)
+      @diagnostic_io.puts(message) if @verbose && @diagnostic_io
+    end
+  end
+end