structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/json/profiles.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "profiles/khoros_api_export"
+
+module StructuredDataToSql
+  module Json
+    # Preparation profiles selectable through the converter's +profile:+
+    # option (CLI --profile). A profile class provides NAME, REQUIRED_SCHEMA,
+    # an InputValidator, .write_preamble(out), and #write(out).
+    module Profiles
+      REGISTRY = { KhorosApiExport::NAME => KhorosApiExport }.freeze
+    end
+  end
+end

data/lib/structured_data_to_sql/json/record_streamer.rb

@@ -0,0 +1,477 @@
+# frozen_string_literal: true
+
+require "oj"
+
+module StructuredDataToSql
+  module Json
+    # Streams the records out of one JSON document without materializing the
+    # whole document: only the envelope (everything outside the records array)
+    # and one record at a time are held in memory.
+    #
+    # Records-array detection, first match wins:
+    #   1. the array at the end of the explicit dot-notation records_path
+    #      (e.g. "data.community.roles.edges"); error if absent
+    #   2. the array under the top-level "data" key
+    #   3. the sole top-level array value (buffered into the envelope first,
+    #      so only suitable for modest non-"data" files; use records_path for
+    #      huge ones)
+    #   4. none: the whole document is a single record
+    #
+    # NDJSON (newline-delimited JSON) support:
+    #   When ndjson: true is passed, or auto-detection determines the stream
+    #   contains multiple top-level JSON documents (one per line), each line is
+    #   parsed independently and records from all lines are yielded in order.
+    #   All lines must share the same structure (same records_path). The
+    #   envelope is taken from the first non-empty line; subsequent lines'
+    #   envelopes are ignored (they are structurally identical page wrappers).
+    class RecordStreamer
+      Result =
+        Struct.new(
+          :envelope,
+          :records_key,
+          :record_count,
+          :truncated,
+          :parse_error,
+          :skipped_lines,
+          keyword_init: true
+        )
+
+      # Maximum bytes read for each line considered during NDJSON probing.
+      NDJSON_PROBE_LINE_BYTES = 16 * 1024 * 1024
+
+      def initialize(records_path: nil)
+        @records_path = records_path
+      end
+
+      # Yields each record (a plain Hash/Array/scalar tree) with its 0-based
+      # ordinal. Returns a Result with the envelope hash (records excluded).
+      #
+      # With recover: true, a parse failure inside the records array returns a
+      # truncated Result covering the records that completed before the
+      # failure (each yielded record is a fully parsed tree, so everything
+      # already yielded is intact); the partial tail record is discarded.
+      # Failures before the first complete record still raise.
+      #
+      # With ndjson: true, each line of +io+ is treated as a separate JSON
+      # document. Auto-detection (ndjson: :auto, the default) probes the first
+      # two non-empty lines; set ndjson: false to disable auto-detection.
+      def each_record(io, recover: false, ndjson: :auto, &block)
+        if ndjson == true || (ndjson == :auto && ndjson_stream?(io))
+          each_record_ndjson(io, recover: recover, &block)
+        else
+          each_record_single(io, recover: recover, &block)
+        end
+      end
+
+      private
+
+      # ---------------------------------------------------------------------------
+      # Single-document path (original behaviour)
+      # ---------------------------------------------------------------------------
+
+      def each_record_single(io, recover: false, &block)
+        path_keys, records_key = path_keys_and_root_key
+        handler =
+          Handler.new(
+            records_key: records_key,
+            records_path: path_keys,
+            on_record: block
+          )
+        begin
+          Oj.sc_parse(handler, io)
+        rescue Oj::ParseError, EncodingError => e
+          unless recover && handler.records_found &&
+                   handler.record_count.positive?
+            raise
+          end
+
+          return(
+            Result.new(
+              envelope: handler.envelope || {},
+              records_key: handler.records_key_used,
+              record_count: handler.record_count,
+              truncated: true,
+              parse_error: e.message.sub(/ \[\S+\]\z/, "")
+            )
+          )
+        end
+        root = handler.root_value
+        if handler.records_found
+          return(
+            Result.new(
+              envelope: handler.envelope || {},
+              records_key: handler.records_key_used,
+              record_count: handler.record_count
+            )
+          )
+        end
+
+        if @records_path
+          raise UsageError,
+                "No array found under records path '#{@records_path}' in JSON input"
+        end
+
+        fallback_records(root, handler, &block)
+      end
+
+      # ---------------------------------------------------------------------------
+      # NDJSON path — one JSON document per line
+      # ---------------------------------------------------------------------------
+
+      def each_record_ndjson(io, recover: false, &block)
+        total_count = 0
+        envelope = nil
+        records_key_used = nil
+        truncated = false
+        parse_error = nil
+        skipped_lines = 0
+        line_number = 0
+
+        io.each_line do |line|
+          line_number += 1
+          next if line.empty?
+          next if line.getbyte(0) <= 32 && line.match?(/\A[[:space:]]*\z/)
+
+          begin
+            root = Oj.strict_load(line)
+          rescue Oj::ParseError, EncodingError => e
+            if recover
+              truncated = true
+              skipped_lines += 1
+              parse_error = e.message.sub(/ \[\S+\]\z/, "")
+              next
+            end
+            raise
+          end
+          page_envelope, page_records_key, records =
+            ndjson_page(root, line_number)
+          records.each do |record|
+            block.call(record, total_count)
+            total_count += 1
+          end
+          envelope ||= page_envelope
+          records_key_used ||= page_records_key
+        end
+
+        if recover && skipped_lines.positive? && total_count.zero?
+          raise UsageError,
+                "No complete records could be recovered from NDJSON input (skipped #{skipped_lines} malformed line(s))"
+        end
+
+        Result.new(
+          envelope: envelope || {},
+          records_key: records_key_used,
+          record_count: total_count,
+          truncated: truncated || nil,
+          parse_error: parse_error,
+          skipped_lines: skipped_lines.positive? ? skipped_lines : nil
+        )
+      end
+
+      # NDJSON lines are complete JSON documents, so Oj's direct strict parser
+      # is substantially cheaper than constructing a streaming handler and IO
+      # wrapper for every line. The resulting page is already bounded by one
+      # line; select its records with the same rules as the streaming path.
+      def ndjson_page(root, line_number)
+        if @records_path
+          path = @records_path.split(".")
+          parent = root
+          path[0...-1].each { |key| parent = parent[key] if parent.is_a?(Hash) }
+          records = parent.delete(path.last) if parent.is_a?(Hash)
+          unless records.is_a?(Array)
+            raise UsageError,
+                  "No array found under records path '#{@records_path}' in JSON input on NDJSON line #{line_number}"
+          end
+          return root, @records_path, records
+        end
+
+        return {}, nil, root if root.is_a?(Array)
+        unless root.is_a?(Hash)
+          raise UsageError,
+                "JSON input is a bare scalar value and cannot be converted"
+        end
+
+        return root, "data", root.delete("data") if root["data"].is_a?(Array)
+
+        array_keys = root.keys.select { |key| root[key].is_a?(Array) }
+        if array_keys.length == 1
+          key = array_keys.first
+          return root, key, root.delete(key)
+        end
+
+        [{}, nil, [root]]
+      end
+
+      # ---------------------------------------------------------------------------
+      # NDJSON auto-detection
+      # ---------------------------------------------------------------------------
+
+      # Returns true if +io+ looks like a newline-delimited multi-document
+      # stream. The probe reads up to two non-empty lines and checks whether the
+      # first is a complete JSON object/array. The IO is rewound to its original
+      # position after probing, so the caller continues from where it started.
+      #
+      # This works for seekable IO objects. CountingIO wraps readpartial/read but
+      # does not expose seek; we reach through to the underlying IO when
+      # available, otherwise we fall back to false (no auto-detection). Forced
+      # ndjson: true does not require seeking.
+      def ndjson_stream?(io)
+        # Unwrap CountingIO so we can seek.
+        raw_io = io.respond_to?(:__raw_io__) ? io.__raw_io__ : io
+        unless raw_io.respond_to?(:seek) && raw_io.respond_to?(:gets) &&
+                 raw_io.respond_to?(:pos)
+          return false
+        end
+
+        original_pos = raw_io.pos
+        raw_io.seek(original_pos)
+        lines = []
+        while lines.length < 2
+          line = raw_io.gets(NDJSON_PROBE_LINE_BYTES)
+          break if line.nil?
+
+          lines << line unless line.strip.empty?
+        end
+        return false if lines.length < 2
+
+        first_line = lines.first
+
+        # The first line must be a syntactically complete JSON object/array.
+        begin
+          first_value = Oj.load(first_line)
+        rescue Oj::ParseError, EncodingError
+          return false
+        end
+        return false unless first_value.is_a?(Hash) || first_value.is_a?(Array)
+
+        # There is more content after that first line.
+        true
+      rescue IOError, SystemCallError
+        false
+      ensure
+        begin
+          raw_io.seek(original_pos) if raw_io && original_pos
+        rescue IOError, SystemCallError
+          nil
+        end
+      end
+
+      # ---------------------------------------------------------------------------
+      # Shared helpers
+      # ---------------------------------------------------------------------------
+
+      def path_keys_and_root_key
+        path_keys = @records_path ? @records_path.split(".") : nil
+        # Single-key path: legacy behaviour — treat it as the flat records key
+        # at depth 1 (same as before this change, so old callers are unaffected).
+        records_key =
+          (path_keys && path_keys.length == 1) ? path_keys.first : "data"
+        [path_keys, records_key]
+      end
+
+      def fallback_records(root, handler, &block)
+        unless root.is_a?(Hash)
+          raise UsageError,
+                "JSON input is a bare scalar value and cannot be converted"
+        end
+
+        envelope = handler.envelope
+
+        array_keys = envelope.keys.select { |key| envelope[key].is_a?(Array) }
+        if array_keys.length == 1
+          key = array_keys.first
+          records = envelope.delete(key)
+          records.each_with_index(&block)
+          Result.new(
+            envelope: envelope,
+            records_key: key,
+            record_count: records.length
+          )
+        else
+          block.call(envelope, 0)
+          Result.new(envelope: {}, records_key: nil, record_count: 1)
+        end
+      end
+
+      # -- Oj::ScHandler callback names
+      class Handler < ::Oj::ScHandler
+        RECORDS = Object.new
+
+        attr_reader :envelope, :records_key_used, :record_count, :root_value
+
+        # +records_key+ is the single-key fallback used to match a depth-1 array
+        # (kept for legacy single-key paths and the default "data" case).
+        # +records_path+ is the full key sequence for multi-level paths; when set,
+        # the handler descends through intermediate objects until the final segment
+        # matches an array — that array becomes the RECORDS stream.
+        def initialize(records_key:, on_record:, records_path: nil)
+          super()
+          @records_key = records_key
+          @records_path = records_path
+          @on_record = on_record
+          @stack = []
+          @pending_key = nil
+          @envelope = nil
+          @records_found = false
+          @records_key_used = nil
+          @record_count = 0
+          @root_value = nil
+          # Number of path segments matched by the current object branch.
+          @path_matched = 0
+          @path_match_stack = []
+          # The most recent key seen at the current path match depth.
+          @path_pending_key = nil
+        end
+
+        def records_found
+          @records_found
+        end
+
+        def hash_start
+          hash = {}
+          previous_path_matched = @path_matched
+          @path_matched += 1 if path_intermediate_value?
+          @path_match_stack.push(previous_path_matched)
+          @envelope = hash if @stack.empty?
+          @stack.push(hash)
+          hash
+        end
+
+        def hash_end
+          obj = @stack.pop
+          @path_matched = @path_match_stack.pop || 0
+          obj
+        end
+
+        def hash_key(key)
+          # Legacy single-level match: track the pending key at depth 1.
+          @pending_key = key if @stack.length == 1
+
+          # Multi-level path: track the key seen at the current match depth,
+          # and advance the match counter for non-final segments.
+          if @records_path && !@records_found
+            @path_pending_key = key if @stack.length == @path_matched + 1
+          end
+
+          key
+        end
+
+        def hash_set(hash, key, value)
+          hash[key] = value unless value.equal?(RECORDS)
+        end
+
+        def array_start
+          target =
+            if @stack.empty?
+              @records_found = true
+              RECORDS
+            elsif multi_level_records_array?
+              @records_found = true
+              @records_key_used = @records_path.join(".")
+              RECORDS
+            elsif @stack.length == 1 && @stack.first.equal?(@envelope) &&
+                  @pending_key == @records_key
+              @records_found = true
+              @records_key_used = @records_key
+              RECORDS
+            else
+              []
+            end
+          @stack.push(target)
+          target
+        end
+
+        def array_end
+          @stack.pop
+        end
+
+        def array_append(array, value)
+          if array.equal?(RECORDS)
+            @on_record.call(value, @record_count)
+            @record_count += 1
+          else
+            array << value
+          end
+        end
+
+        def add_value(value)
+          @root_value = value
+          @envelope = {} if @envelope.nil?
+        end
+
+        private
+
+        # A key matching an intermediate path segment only counts after its
+        # value actually starts as an object. This keeps partial matches from
+        # leaking into sibling branches or across scalar/array values.
+        def path_intermediate_value?
+          return false unless @records_path && !@records_found
+          return false if @stack.empty?
+          return false if @path_matched >= @records_path.length - 1
+
+          @stack.length == @path_matched + 1 &&
+            @path_pending_key == @records_path[@path_matched]
+        end
+
+        # Returns true when the multi-level path has been fully matched and the
+        # current pending_key is the final segment — meaning this array_start
+        # call is the records array.
+        def multi_level_records_array?
+          return false unless @records_path && !@records_found
+
+          # All path segments except the last have been matched via hash keys;
+          # the current @path_pending_key is the final segment.
+          @path_matched == @records_path.length - 1 &&
+            @path_pending_key == @records_path.last &&
+            @stack.length == @path_matched + 1
+        end
+      end
+      # rubocop:enable Naming/MethodName
+    end
+
+    # IO wrapper reporting consumed bytes; Oj.sc_parse drives custom IO-like
+    # objects through readpartial.
+    class CountingIO
+      def initialize(io, &on_bytes)
+        @io = io
+        @on_bytes = on_bytes
+      end
+
+      # Exposed so RecordStreamer can reach the underlying IO for seek-based
+      # NDJSON probe/rewind without going through the counting wrapper.
+      def __raw_io__
+        @io
+      end
+
+      def readpartial(max_length, out_buffer = nil)
+        chunk =
+          (
+            if out_buffer
+              @io.readpartial(max_length, out_buffer)
+            else
+              @io.readpartial(max_length)
+            end
+          )
+        @on_bytes.call(chunk.bytesize) if chunk
+        chunk
+      end
+
+      def read(length = nil, out_buffer = nil)
+        chunk = out_buffer ? @io.read(length, out_buffer) : @io.read(length)
+        @on_bytes.call(chunk.bytesize) if chunk
+        chunk
+      end
+
+      # Delegates each_line to the underlying IO, reporting bytes as each line
+      # is yielded so progress tracking still works during NDJSON iteration.
+      def each_line(&block)
+        return enum_for(:each_line) unless block
+
+        @io.each_line do |line|
+          @on_bytes.call(line.bytesize)
+          block.call(line)
+        end
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/json/schema_inferrer.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "time"
+
+module StructuredDataToSql
+  module Json
+    ColumnDef =
+      Struct.new(:name, :kind, :sql_type, :null, :comment, keyword_init: true)
+
+    # Accumulates per-column type observations across all records of a table
+    # and finalizes them into MySQL column definitions via a widening lattice:
+    # unknown -> boolean | integer -> float | datetime -> string (sized into
+    # VARCHAR/TEXT/MEDIUMTEXT/LONGTEXT by max byte length).
+    class ColumnProfile
+      ISO_DATETIME =
+        /\A\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:?\d{2})?\z/
+
+      attr_reader :kind, :seen_count
+
+      def initialize(raw_dates: false)
+        @raw_dates = raw_dates
+        @kind = :unknown
+        @max_bytes = 0
+        @null_seen = false
+        @seen_count = 0
+      end
+
+      def observe(value)
+        @seen_count += 1
+        case value
+        when nil
+          @null_seen = true
+        when JsonValue
+          @kind = :json
+          @max_bytes = [@max_bytes, value.text.bytesize].max
+        when true, false
+          widen(:boolean)
+        when Integer
+          widen(:integer)
+        when Float
+          widen(:float)
+        when String
+          @max_bytes = [@max_bytes, value.bytesize].max
+          widen(!@raw_dates && value.match?(ISO_DATETIME) ? :datetime : :string)
+        else
+          @max_bytes = [@max_bytes, value.to_s.bytesize].max
+          widen(:string)
+        end
+      end
+
+      def finalize(name, row_count)
+        ColumnDef.new(
+          name: name,
+          kind: @kind,
+          sql_type: sql_type,
+          null: @null_seen || @seen_count < row_count
+        )
+      end
+
+      private
+
+      def widen(observed)
+        return @kind = :json if @kind == :json || observed == :json
+        return @kind = observed if @kind == :unknown
+        return if @kind == observed
+
+        numeric = [@kind, observed].sort == %i[float integer]
+        @kind = numeric ? :float : :string
+      end
+
+      def sql_type
+        case @kind
+        when :boolean
+          "TINYINT(1)"
+        when :integer
+          "BIGINT"
+        when :float
+          "DOUBLE"
+        when :datetime
+          "DATETIME"
+        when :json
+          "LONGTEXT"
+        else
+          string_type
+        end
+      end
+
+      def string_type
+        if @max_bytes <= 255
+          "VARCHAR(255)"
+        elsif @max_bytes <= 65_535
+          "TEXT"
+        elsif @max_bytes <= 16_777_215
+          "MEDIUMTEXT"
+        else
+          "LONGTEXT"
+        end
+      end
+    end
+
+    # Column profiles for one table, in first-observation order.
+    class TableProfile
+      attr_reader :table_rel, :row_count
+
+      def initialize(table_rel, raw_dates: false)
+        @table_rel = table_rel
+        @raw_dates = raw_dates
+        @columns = {}
+        @row_count = 0
+      end
+
+      def observe_row(columns)
+        @row_count += 1
+        columns.each do |name, value|
+          (@columns[name] ||= ColumnProfile.new(raw_dates: @raw_dates)).observe(
+            value
+          )
+        end
+      end
+
+      def child?
+        !@table_rel.empty?
+      end
+
+      def finalize
+        @columns.map { |name, profile| profile.finalize(name, @row_count) }
+      end
+    end
+
+    # Collects TableProfiles for every table produced by shredding a file.
+    class SchemaInferrer
+      attr_reader :profiles
+
+      def initialize(raw_dates: false)
+        @raw_dates = raw_dates
+        @profiles = {}
+      end
+
+      def observe(row)
+        (
+          @profiles[row.table_rel] ||= TableProfile.new(
+            row.table_rel,
+            raw_dates: @raw_dates
+          )
+        ).observe_row(row.columns)
+        row.children.each { |child| observe(child) }
+      end
+    end
+  end
+end