structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/diagnostics_report.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "pathname"
+
+require_relative "format"
+require_relative "io_support"
+require_relative "version"
+
+module StructuredDataToSql
+  # Writes the Markdown diagnostics report that sits beside a dump: what the
+  # run processed, every diagnostic with its complete lists and suggested
+  # action, the exporter runs that produced the source (when a manifest was
+  # found), and the per-file ledger. Written atomically and always — a clean
+  # run replaces a stale report with a "no warnings" one. Never includes
+  # member values, credentials, or per-row error payloads.
+  module DiagnosticsReport
+    module_function
+
+    def default_path(output)
+      return nil unless output.is_a?(String) || output.is_a?(Pathname)
+      return nil if output.to_s == "-"
+
+      "#{output}.diagnostics.md"
+    end
+
+    # +context+ keys: format, source, output, profile, started_at,
+    # finished_at, elapsed, stats (Hash), diagnostics (Array<Diagnostic>),
+    # ledger (Array<Hash name,size,rows,child_rows,elapsed>), manifest
+    # (Json::ExporterManifest or nil), source_paths (Hash table => path).
+    def write(path, context)
+      pathname = Pathname(path)
+      temporary = IOSupport.unique_adjacent_path(pathname)
+      begin
+        File.write(temporary, render(context))
+        File.rename(temporary, pathname)
+      ensure
+        FileUtils.rm_f(temporary) if temporary.exist?
+      end
+      pathname.to_s
+    rescue SystemCallError, IOError => e
+      raise OutputError,
+            "Could not write diagnostics report #{path}: #{e.message}"
+    end
+
+    def render(context)
+      stats = context[:stats] || {}
+      diagnostics = Array(context[:diagnostics])
+      lines = []
+      lines << "# Conversion diagnostics"
+      lines << ""
+      lines << "| | |"
+      lines << "|---|---|"
+      lines << "| Tool | structured_data_to_sql #{VERSION} (#{context[:format]}) |"
+      lines << "| Source | `#{context[:source]}` |"
+      lines << "| Output | `#{context[:output]}` |"
+      lines << "| Profile | #{context[:profile] || "none"} |"
+      lines << "| Started | #{context[:started_at]} |" if context[:started_at]
+      if context[:finished_at]
+        lines << "| Finished | #{context[:finished_at]} |"
+      end
+      if context[:elapsed]
+        lines << "| Elapsed | #{Format.format_duration(context[:elapsed])} |"
+      end
+      lines << ""
+      lines.concat(
+        render_summary(stats, diagnostics.length, context[:input_bytes])
+      )
+      lines.concat(render_diagnostics(diagnostics))
+      lines.concat(render_manifest(context[:manifest], context[:source_paths]))
+      lines.concat(render_ledger(context[:ledger]))
+      "#{lines.join("\n")}\n"
+    end
+
+    def render_summary(stats, warning_count, input_bytes = nil)
+      lines = ["## Run summary", ""]
+      lines << "- Files processed: #{Format.format_count(stats[:files_processed])}" \
+        "#{stats[:files_skipped].to_i.positive? ? " (#{Format.format_count(stats[:files_skipped])} skipped)" : ""}"
+      lines << "- Rows: #{Format.format_count(stats[:rows_processed])}" \
+        "#{stats[:child_rows_processed].to_i.positive? ? " (+#{Format.format_count(stats[:child_rows_processed])} child rows)" : ""}"
+      lines << "- Tables: #{Format.format_count(stats[:tables_processed])}" \
+        "#{stats[:tables_skipped].to_i.positive? ? " (#{Format.format_count(stats[:tables_skipped])} skipped)" : ""}"
+      if input_bytes
+        lines << "- Input: #{Format.format_size(input_bytes)} · Read: #{Format.format_size(stats[:bytes_read])} (all passes) · Written: #{Format.format_size(stats[:bytes_written])}"
+      elsif stats[:bytes_read]
+        lines << "- Read: #{Format.format_size(stats[:bytes_read])} · Written: #{Format.format_size(stats[:bytes_written])}"
+      end
+      lines << "- Warnings: #{warning_count}"
+      lines << ""
+      lines
+    end
+
+    def render_diagnostics(diagnostics)
+      lines = ["## Diagnostics", ""]
+      if diagnostics.empty?
+        lines << "No warnings were raised by this conversion."
+        lines << ""
+        return lines
+      end
+
+      diagnostics.each_with_index do |diagnostic, index|
+        heading = "### #{index + 1}. #{diagnostic.title}"
+        heading +=
+          " (#{Format.format_count(diagnostic.count)})" if diagnostic.count
+        lines << heading
+        lines << ""
+        lines << "`#{diagnostic.code}`"
+        lines << ""
+        if diagnostic.summary
+          lines << diagnostic.summary
+          lines << ""
+        end
+        diagnostic.details.each { |detail| lines << "- #{detail}" }
+        lines << "" if diagnostic.details.any?
+        if diagnostic.items.any?
+          lines << "**#{diagnostic.items_label || "Items"}** (#{Format.format_count(diagnostic.items.length)}):"
+          lines << ""
+          diagnostic.items.each { |item| lines << "- `#{item}`" }
+          lines << ""
+        end
+        if diagnostic.action
+          lines << "**Action:** #{diagnostic.action}"
+          lines << ""
+        end
+        if diagnostic.sources.any?
+          lines << "**Sources:** #{diagnostic.sources.map { |source| "`#{source}`" }.join(", ")}"
+          lines << ""
+        end
+        lines << "<details><summary>Warning line</summary>"
+        lines << ""
+        lines << "```"
+        lines << diagnostic.message
+        lines << "```"
+        lines << ""
+        lines << "</details>"
+        lines << ""
+      end
+      lines
+    end
+
+    def render_manifest(manifest, source_paths)
+      return [] if manifest.nil?
+
+      lines = ["## Exporter runs", ""]
+      lines << "Manifest: `#{manifest.path}`"
+      lines << ""
+      if manifest.problem
+        lines << "Not usable for run scoping: #{manifest.problem}."
+        lines << ""
+        return lines
+      end
+
+      lines << "Quality signals are scoped to the latest run whose manifest entry is `completed`, " \
+        "by attributing the last _N_ rows of the append-only errors/gaps files to it (_N_ = that run's declared count). " \
+        "Rows beyond the sum of declared counts belong to runs that never finalized their entry."
+      lines << ""
+      lines << "| Run | Status | Started | Finished | Mode | Credential | Emails | Errors | Gaps |"
+      lines << "|---:|---|---|---|---|---|---|---:|---:|"
+      manifest.runs.each do |run|
+        lines << "| #{run.index} | #{run.status} | #{run.started_at} | #{run.finished_at} | #{run.mode} | " \
+          "#{run.credential} | #{run.user_emails} | #{Format.format_count(run.errors)} | #{Format.format_count(run.gaps)} |"
+      end
+      lines << ""
+      Array(source_paths).each do |table, path|
+        lines << "- Raw `#{table}` rows: `#{path}`" if path
+      end
+      lines << "" if source_paths&.any?
+      lines
+    end
+
+    def render_ledger(ledger)
+      ledger = Array(ledger)
+      return [] if ledger.empty?
+
+      lines = ["## Files", ""]
+      lines << "| File | Size | Rows | Child rows | Time |"
+      lines << "|---|---:|---:|---:|---:|"
+      ledger.each do |entry|
+        lines << "| #{entry[:name]} | #{Format.format_size(entry[:size])} | " \
+          "#{Format.format_count(entry[:rows])} | #{Format.format_count(entry[:child_rows])} | " \
+          "#{Format.format_duration(entry[:elapsed])} |"
+      end
+      lines << ""
+      lines
+    end
+  end
+end

data/lib/structured_data_to_sql/errors.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  class Error < StandardError
+  end
+
+  class ConfigurationError < Error
+  end
+  class InputError < Error
+  end
+  class ParseError < Error
+  end
+  class OutputError < Error
+  end
+
+  # Kept as a compatibility name for the pre-public extraction.
+  UsageError = ConfigurationError
+
+  module Json
+    class Error < StructuredDataToSql::ConfigurationError
+    end
+    class ConfigurationError < Error
+    end
+    class InputError < Error
+    end
+    class ParseError < Error
+    end
+    class OutputError < Error
+    end
+  end
+
+  module MysqlDumpXml
+    class Error < StructuredDataToSql::ConfigurationError
+    end
+    class ConfigurationError < Error
+    end
+    class InputError < Error
+    end
+    class ParseError < Error
+    end
+    class OutputError < Error
+    end
+  end
+
+  # The public XML namespace. The implementation name remains precise because
+  # this format accepts mysqldump XML rather than arbitrary XML documents.
+  Xml = MysqlDumpXml
+end

data/lib/structured_data_to_sql/format.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module StructuredDataToSql
+  module Format
+    module_function
+
+    def format_size(size)
+      size = size.to_i
+      if size >= 1024 * 1024 * 1024
+        return format("%.1f GB", size / 1024.0 / 1024.0 / 1024.0)
+      end
+      return format("%.1f MB", size / 1024.0 / 1024.0) if size >= 1024 * 1024
+      return format("%.1f KB", size / 1024.0) if size >= 1024
+
+      "#{size} B"
+    end
+
+    def format_count(count)
+      count.to_i.to_s.reverse.scan(/\d{1,3}/).join(",").reverse
+    end
+
+    def format_duration(seconds)
+      seconds = seconds.to_f
+      return format("%.1fs", seconds) if seconds < 60
+
+      total = seconds.round
+      hours, remainder = total.divmod(3600)
+      minutes, secs = remainder.divmod(60)
+      return format("%dh%02dm", hours, minutes) if hours.positive?
+
+      format("%dm%02ds", minutes, secs)
+    end
+
+    def format_rate(bytes, seconds)
+      seconds = seconds.to_f
+      return nil if seconds <= 0
+
+      "#{format_size(bytes.to_f / seconds)}/s"
+    end
+  end
+end

data/lib/structured_data_to_sql/io_support.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "securerandom"
+require "tempfile"
+require "zlib"
+require "fileutils"
+
+module StructuredDataToSql
+  module IOSupport
+    class CountingWriter
+      attr_reader :bytes_written
+
+      def initialize(io)
+        @io = io
+        @bytes_written = 0
+      end
+
+      def write(data)
+        written = @io.write(data)
+        @bytes_written += written
+        written
+      end
+
+      def method_missing(name, *args, &block)
+        @io.public_send(name, *args, &block)
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        @io.respond_to?(name, include_private) || super
+      end
+    end
+
+    module_function
+
+    def readable_io?(object)
+      object.respond_to?(:read) && !object.is_a?(String) &&
+        !object.is_a?(Pathname)
+    end
+
+    def writable_io?(object)
+      object.respond_to?(:write) && !object.is_a?(String) &&
+        !object.is_a?(Pathname)
+    end
+
+    def discover(source, patterns:, stream_extension:, input_gzip: false)
+      temporary_files = []
+      source_items = readable_io?(source) ? [source] : Array(source)
+      candidates =
+        source_items.flat_map do |item|
+          if readable_io?(item)
+            suffix = "#{stream_extension}#{input_gzip ? ".gz" : ""}"
+            tempfile = Tempfile.new(["structured-data-to-sql-input-", suffix])
+            tempfile.binmode
+            IO.copy_stream(item, tempfile)
+            tempfile.close
+            temporary_files << tempfile
+            Pathname(tempfile.path)
+          else
+            path = Pathname(item)
+            raise InputError, "Input not found: #{path}" unless path.exist?
+
+            if path.directory?
+              patterns.flat_map { |pattern| path.glob(pattern).to_a }
+            else
+              path
+            end
+          end
+        end
+      [candidates.flatten.uniq.sort_by(&:to_s), temporary_files]
+    rescue SystemCallError, IOError => e
+      temporary_files&.each(&:unlink)
+      raise InputError, e.message
+    end
+
+    # Bytes the streaming reader will hand out for +path+: the file size for
+    # plain files, the gzip ISIZE trailer (uncompressed length modulo 2**32,
+    # lifted until it is at least the compressed size) for .gz files. One
+    # four-byte read per file, no decompression, never raises.
+    def estimated_input_bytes(path)
+      pathname = Pathname(path)
+      size = pathname.size
+      return size unless pathname.extname == ".gz" && size >= 18
+
+      trailer = File.binread(pathname.to_s, 4, size - 4)
+      estimate = trailer.unpack1("V")
+      estimate += 2**32 while estimate < size
+      estimate
+    rescue StandardError
+      begin
+        Pathname(path).size
+      rescue StandardError
+        0
+      end
+    end
+
+    def unique_adjacent_path(path)
+      pathname = Pathname(path)
+      pathname.dirname.join(
+        ".#{pathname.basename}.tmp-#{Process.pid}-#{SecureRandom.hex(8)}"
+      )
+    end
+
+    def validate_output_target!(target)
+      return if writable_io?(target)
+
+      path = Pathname(target)
+      return unless path.directory?
+
+      raise OutputError,
+            "Output path is a directory; provide a file path: #{path}"
+    end
+
+    def with_output(target, gzip:, atomic: true)
+      validate_output_target!(target)
+      if writable_io?(target)
+        counting_target = CountingWriter.new(target)
+        if gzip
+          writer = Zlib::GzipWriter.new(counting_target)
+          begin
+            yield writer
+          ensure
+            writer.finish
+          end
+        else
+          yield counting_target
+        end
+        return counting_target.bytes_written
+      end
+
+      path = Pathname(target)
+      temporary_path = atomic ? unique_adjacent_path(path) : path
+      begin
+        if gzip
+          Zlib::GzipWriter.open(temporary_path.to_s) do |gzip_writer|
+            yield gzip_writer
+          end
+        else
+          File.open(temporary_path, "w:utf-8") do |file_writer|
+            yield file_writer
+          end
+        end
+        File.rename(temporary_path, path) if atomic
+        File.size(path)
+      rescue SystemCallError, IOError, Zlib::Error => e
+        raise OutputError, e.message
+      ensure
+        if atomic && temporary_path && temporary_path.exist?
+          FileUtils.rm_f(temporary_path)
+        end
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/json/exporter_manifest.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "json"
+require "pathname"
+
+module StructuredDataToSql
+  module Json
+    # Reads the Khoros exporter's manifest.json (kept beside the ndjson/
+    # directory) just far enough to scope quality signals to the latest run.
+    # Everything is best-effort: a missing or unparsable manifest yields an
+    # instance with +problem+ set and no runs; nothing here ever raises.
+    class ExporterManifest
+      Run =
+        Struct.new(
+          :index,
+          :status,
+          :started_at,
+          :finished_at,
+          :mode,
+          :credential,
+          :user_emails,
+          :errors,
+          :gaps,
+          keyword_init: true
+        ) do
+          def completed?
+            status == "completed"
+          end
+        end
+
+      FILENAME = "manifest.json"
+
+      attr_reader :path, :runs, :problem
+
+      # Looks in the source directory itself and in its parent, so both
+      # `export/ndjson` and `export/` work as the conversion source.
+      def self.locate(source)
+        source = source.first if source.is_a?(Array)
+        return nil unless source.is_a?(String) || source.is_a?(Pathname)
+
+        base = Pathname(source)
+        base = base.dirname if base.file?
+        return nil unless base.directory?
+
+        [base, base.parent].each do |directory|
+          candidate = directory.join(FILENAME)
+          return candidate if candidate.file?
+        end
+        nil
+      rescue SystemCallError
+        nil
+      end
+
+      def self.load(path)
+        return nil if path.nil?
+
+        new(path)
+      end
+
+      def initialize(path)
+        @path = Pathname(path)
+        @runs = []
+        @problem = nil
+        parse
+      end
+
+      def latest_completed
+        @runs.reverse.find(&:completed?)
+      end
+
+      def errors_total
+        @runs.sum(&:errors)
+      end
+
+      def gaps_total
+        @runs.sum(&:gaps)
+      end
+
+      private
+
+      def parse
+        document = JSON.parse(File.read(@path.to_s))
+        runs = document.is_a?(Hash) ? document["runs"] : nil
+        unless runs.is_a?(Array)
+          @problem = "manifest has no runs array"
+          return
+        end
+
+        @runs =
+          runs.each_with_index.map do |run, index|
+            run = {} unless run.is_a?(Hash)
+            capabilities =
+              run["capabilities"].is_a?(Hash) ? run["capabilities"] : {}
+            options = run["options"].is_a?(Hash) ? run["options"] : {}
+            counts = run["counts"].is_a?(Hash) ? run["counts"] : {}
+            Run.new(
+              index: index,
+              status: run["status"].to_s,
+              started_at: run["started_at"],
+              finished_at: run["finished_at"],
+              mode: options["mode"],
+              credential: capabilities["credential"],
+              user_emails: capabilities["user_emails"],
+              errors: integer_of(run["errors"]),
+              gaps: integer_of(counts["gaps"])
+            )
+          end
+      rescue JSON::ParserError, SystemCallError, IOError, EncodingError => e
+        @runs = []
+        @problem = "manifest unreadable: #{e.message[0, 120]}"
+      end
+
+      def integer_of(value)
+        case value
+        when Integer
+          value
+        when Array
+          value.length
+        when Hash
+          value.values.sum { |item| integer_of(item) }
+        else
+          0
+        end
+      end
+    end
+  end
+end