structured_data_to_sql 0.1.0

data/lib/structured_data_to_sql/json/shredder.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require "digest"
+require "oj"
+
+module StructuredDataToSql
+  module Json
+    # A JSON value kept verbatim as serialized JSON text (fallback for shapes
+    # that do not shred cleanly into columns or child tables).
+    JsonValue = Struct.new(:text)
+
+    # One shredded record: +table_rel+ is the table's path relative to the
+    # root table ("" for the root itself), +columns+ maps final column names
+    # to raw values, +children+ holds nested ShreddedRows for child tables.
+    ShreddedRow =
+      Struct.new(:table_rel, :columns, :children, :ordinal, :parent_natural_id)
+
+    # Allocates deterministic, collision-free table and column names. Built
+    # during the inference pass and reused verbatim during the emit pass so
+    # both passes agree on every name.
+    class NameRegistry
+      MAX_NAME_LENGTH = 64
+
+      attr_reader :collisions
+
+      def initialize
+        @columns = {}
+        @column_owner = Hash.new { |hash, key| hash[key] = {} }
+        @children = {}
+        @child_owner = Hash.new { |hash, key| hash[key] = {} }
+        @collisions = 0
+      end
+
+      def column_name(table_rel, raw_path, representation)
+        key = [table_rel, representation, raw_path.join("\0")]
+        @columns[key] ||= allocate(@column_owner[table_rel], key) do
+          base = sanitize_path(raw_path)
+          representation == :json ? "#{base}_json" : base
+        end
+      end
+
+      def child_rel(table_rel, raw_path)
+        key = [table_rel, raw_path.join("\0")]
+        @children[key] ||= allocate(@child_owner[:tables], key) do
+          rel = sanitize_path(raw_path)
+          table_rel.empty? ? rel : "#{table_rel}_#{rel}"
+        end
+      end
+
+      def self.sanitize(name)
+        sanitized =
+          name
+            .to_s
+            .gsub(/([a-z0-9])([A-Z])/, '\1_\2')
+            .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+            .downcase
+            .gsub(/[^a-z0-9_]+/, "_")
+            .squeeze("_")
+            .gsub(/\A_+|_+\z/, "")
+        sanitized = "t_#{sanitized}" if sanitized.match?(/\A\d/)
+        sanitized.empty? ? "unnamed" : sanitized
+      end
+
+      def self.truncate(name)
+        return name if name.length <= MAX_NAME_LENGTH
+
+        "#{name[0, MAX_NAME_LENGTH - 9]}_#{Digest::SHA1.hexdigest(name)[0, 8]}"
+      end
+
+      private
+
+      def sanitize_path(raw_path)
+        self.class.truncate(
+          raw_path.map { |part| self.class.sanitize(part) }.join("_")
+        )
+      end
+
+      def allocate(taken, key)
+        candidate = self.class.truncate(yield)
+        unless taken[candidate].nil? || taken[candidate] == key
+          @collisions += 1
+          suffix = 2
+          suffix += 1 until taken[
+            self.class.truncate("#{candidate}_#{suffix}")
+          ].nil?
+          candidate = self.class.truncate("#{candidate}_#{suffix}")
+        end
+        taken[candidate] = key
+        candidate
+      end
+    end
+
+    # Turns one JSON record into a tree of ShreddedRows following the
+    # convention-based rules: nested objects flatten into prefixed columns,
+    # arrays become child tables, GraphQL edges/node wrappers unwrap, and
+    # anything too deep or irregular falls back to a *_json text column.
+    class Shredder
+      def initialize(
+        registry:,
+        max_depth: 5,
+        graphql_unwrap: true,
+        json_column_paths: nil,
+        forced_json_paths: nil
+      )
+        @registry = registry
+        @max_depth = max_depth
+        @graphql_unwrap = graphql_unwrap
+        @json_column_paths = (json_column_paths || []).to_set
+        @forced_json_paths = forced_json_paths || Set.new
+        @json_fallbacks = 0
+      end
+
+      attr_reader :json_fallbacks
+
+      def shred(record, ordinal)
+        build_row("", record, ordinal, nil)
+      end
+
+      private
+
+      def build_row(table_rel, record, ordinal, parent_natural_id)
+        row = ShreddedRow.new(table_rel, {}, [], ordinal, parent_natural_id)
+        case record
+        when Hash
+          walk(record, [], row, record)
+        when Array
+          add_json_column(row, ["value"], record)
+        else
+          row.columns["value"] = record
+        end
+        row
+      end
+
+      def walk(object, path, row, natural_id_source)
+        object.each do |key, value|
+          new_path = path + [key]
+          if forced_json?(row.table_rel, new_path)
+            add_json_column(row, new_path, value)
+            next
+          end
+
+          case value
+          when Hash
+            if @graphql_unwrap && (nodes = unwrap_edges(value))
+              handle_array(nodes, new_path, row, natural_id_source)
+            elsif new_path.length >= @max_depth
+              add_json_column(row, new_path, value)
+            else
+              walk(value, new_path, row, natural_id_source)
+            end
+          when Array
+            handle_array(value, new_path, row, natural_id_source)
+          else
+            row.columns[
+              @registry.column_name(row.table_rel, new_path, :scalar)
+            ] = value
+          end
+        end
+      end
+
+      def handle_array(array, path, row, natural_id_source)
+        return if array.empty?
+        return add_json_column(row, path, array) if path.length >= @max_depth
+
+        hashes, others = array.partition { |element| element.is_a?(Hash) }
+        if hashes.any? && others.any? ||
+             array.any? { |element| element.is_a?(Array) }
+          add_json_column(row, path, array)
+        elsif hashes.any?
+          rel = @registry.child_rel(row.table_rel, path)
+          natural_id = natural_id_of(natural_id_source)
+          array.each_with_index do |element, index|
+            row.children << build_row(rel, element, index, natural_id)
+          end
+        else
+          rel = @registry.child_rel(row.table_rel, path)
+          natural_id = natural_id_of(natural_id_source)
+          array.each_with_index do |element, index|
+            row.children << ShreddedRow.new(
+              rel,
+              { "value" => element },
+              [],
+              index,
+              natural_id
+            )
+          end
+        end
+      end
+
+      def add_json_column(row, path, value)
+        @json_fallbacks += 1
+        row.columns[
+          @registry.column_name(row.table_rel, path, :json)
+        ] = JsonValue.new(Oj.dump(value, mode: :compat))
+      end
+
+      def forced_json?(table_rel, path)
+        return true if @forced_json_paths.include?([table_rel, path.join("\0")])
+        return false if @json_column_paths.empty? || !table_rel.empty?
+
+        @json_column_paths.include?(
+          path.map { |part| NameRegistry.sanitize(part) }.join(".")
+        )
+      end
+
+      # Unwraps a Relay-style connection object to the array of node values.
+      #
+      # Tolerates the standard Relay connection shape where sibling keys like
+      # +pageInfo+ and +totalCount+ live alongside +edges+, and where each edge
+      # may carry a +cursor+ (or any other per-edge keys) in addition to +node+.
+      # Only the +node+ value is extracted from each edge; all other edge keys
+      # are silently discarded.
+      #
+      # Returns nil if the object does not look like a connection (no +edges+
+      # key, non-array edges, or any edge that is missing a +node+ key).
+      def unwrap_edges(value)
+        return nil unless value.key?("edges") && value["edges"].is_a?(Array)
+        unless value["edges"].all? { |edge|
+                 edge.is_a?(Hash) && edge.key?("node")
+               }
+          return nil
+        end
+
+        value["edges"].map { |edge| edge["node"] }
+      end
+
+      def natural_id_of(record)
+        id = record["id"]
+        return id.to_s if id.is_a?(String) || id.is_a?(Integer)
+
+        record.each do |key, value|
+          next unless value.is_a?(String) || value.is_a?(Integer)
+
+          sanitized = NameRegistry.sanitize(key)
+          return value.to_s if sanitized == "id" || sanitized.end_with?("_id")
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/structured_data_to_sql/json/sql_emitter.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "time"
+require_relative "../sql_text"
+
+module StructuredDataToSql
+  module Json
+    # Writes MySQL/MariaDB DDL and batched INSERT statements following the
+    # same conventions as the XML converter output.
+    class SqlEmitter
+      META_ROOT = ["_sid"].freeze
+      META_CHILD = %w[_sid _parent_sid _parent_id _ordinal].freeze
+      MAX_INLINE_ROW_BYTES = 60_000
+      TEXT_POINTER_BYTES = 20
+
+      def write_header(out, source)
+        out.write("-- Generated by json-to-sql converter\n")
+        out.write("-- Source: #{source}\n")
+        out.write(
+          "-- Generated at: #{Time.now.strftime("%Y-%m-%d %H:%M:%S")}\n\n"
+        )
+        out.write(
+          "SET NAMES utf8mb4;\nSET sql_mode = '';\nSET FOREIGN_KEY_CHECKS = 0;\nSET UNIQUE_CHECKS = 0;\n\n"
+        )
+      end
+
+      def write_footer(out)
+        out.write("\nSET FOREIGN_KEY_CHECKS = 1;\nSET UNIQUE_CHECKS = 1;\n")
+      end
+
+      def write_create_table(out, name, column_defs, child:)
+        column_defs = row_safe_column_defs(column_defs, child:)
+        lines = ["  `_sid` BIGINT NOT NULL"]
+        if child
+          lines << "  `_parent_sid` BIGINT NOT NULL"
+          lines << "  `_parent_id` VARCHAR(255)"
+          lines << "  `_ordinal` BIGINT NOT NULL"
+        end
+        column_defs.each do |col|
+          definition =
+            "  #{SqlText.escape_identifier(col.name)} #{col.sql_type}"
+          definition += " NOT NULL" unless col.null
+          definition +=
+            " COMMENT #{SqlText.escape_sql_string(col.comment)}" if col.comment
+          lines << definition
+        end
+        lines << "  PRIMARY KEY (`_sid`)"
+        if child
+          lines << "  KEY `idx_parent_sid` (`_parent_sid`)"
+          lines << "  KEY `idx_parent_id` (`_parent_id`)"
+        end
+        id_column = column_defs.find { |col| col.name == "id" }
+        # TEXT-family columns cannot carry a plain KEY without a prefix length.
+        if id_column && !id_column.sql_type.end_with?("TEXT")
+          lines << "  KEY `idx_id` (`id`)"
+        end
+
+        out.write("\n-- Table: #{name}\n")
+        out.write("DROP TABLE IF EXISTS #{SqlText.escape_identifier(name)};\n")
+        out.write(
+          "CREATE TABLE #{SqlText.escape_identifier(name)} (\n#{lines.join(",\n")}\n"
+        )
+        out.write(
+          ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC;\n"
+        )
+      end
+
+      def write_batch(out, name, columns, value_rows)
+        return if value_rows.empty?
+
+        identifiers = columns.map { |column| SqlText.escape_identifier(column) }
+        out.write(
+          "INSERT INTO #{SqlText.escape_identifier(name)} (#{identifiers.join(", ")}) VALUES\n"
+        )
+        out.write(
+          value_rows.map { |values| "  (#{values.join(", ")})" }.join(",\n")
+        )
+        out.write(";\n")
+      end
+
+      # PII manifest derived from x-pii annotations in the JSON Schemas, so
+      # downstream redaction tooling can query column sensitivity in MariaDB.
+      def write_meta_table(out, rows)
+        out.write("\n-- Table: _json_meta\n")
+        out.write("DROP TABLE IF EXISTS `_json_meta`;\n")
+        out.write(<<~SQL)
+          CREATE TABLE `_json_meta` (
+            `table_name` VARCHAR(64) NOT NULL,
+            `column_name` VARCHAR(64) NOT NULL,
+            `json_path` VARCHAR(255) NOT NULL,
+            `pii` TINYINT(1) NOT NULL DEFAULT 0,
+            `pii_note` TEXT,
+            `source_schema` VARCHAR(255),
+            PRIMARY KEY (`table_name`, `column_name`)
+          ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+        SQL
+        value_rows =
+          rows.map do |table, column, json_path, pii, note, source|
+            [
+              SqlText.escape_sql_string(table),
+              SqlText.escape_sql_string(column),
+              SqlText.escape_sql_string(json_path),
+              pii.to_s,
+              note.nil? ? "NULL" : SqlText.escape_sql_string(note),
+              SqlText.escape_sql_string(source)
+            ]
+          end
+        write_batch(
+          out,
+          "_json_meta",
+          %w[table_name column_name json_path pii pii_note source_schema],
+          value_rows
+        )
+      end
+
+      def format_value(value, column_def)
+        return format_null(column_def) if value.nil?
+        return SqlText.escape_sql_string(value.text) if value.is_a?(JsonValue)
+
+        case column_def&.kind
+        when :boolean
+          value == true ? "1" : "0"
+        when :integer, :float
+          if value.is_a?(Numeric)
+            value.to_s
+          else
+            SqlText.escape_sql_string(value.to_s)
+          end
+        when :datetime
+          format_datetime(value)
+        else
+          SqlText.escape_sql_string(value.to_s)
+        end
+      end
+
+      private
+
+      def row_safe_column_defs(column_defs, child:)
+        definitions = column_defs.map(&:dup)
+        inline_bytes = child ? 8 + 8 + 1_022 + 8 : 8
+        inline_bytes +=
+          definitions.sum { |column| inline_bytes_for(column.sql_type) }
+        return definitions if inline_bytes <= MAX_INLINE_ROW_BYTES
+
+        candidates =
+          definitions
+            .select { |column| column.sql_type.match?(/\AVARCHAR\(\d+\)\z/) }
+            .sort_by do |column|
+              width = inline_bytes_for(column.sql_type)
+              [
+                column.name == "id" ? 1 : 0,
+                column.null ? 0 : 1,
+                -width,
+                column.name
+              ]
+            end
+        candidates.each do |column|
+          break if inline_bytes <= MAX_INLINE_ROW_BYTES
+
+          inline_bytes -= inline_bytes_for(column.sql_type) - TEXT_POINTER_BYTES
+          column.sql_type = "TEXT"
+        end
+        definitions
+      end
+
+      def inline_bytes_for(sql_type)
+        if (match = sql_type.match(/\AVARCHAR\((\d+)\)\z/))
+          match[1].to_i * 4 + 2
+        elsif sql_type.end_with?("TEXT")
+          TEXT_POINTER_BYTES
+        elsif sql_type == "BIGINT" || sql_type == "DOUBLE"
+          8
+        elsif sql_type == "DATETIME"
+          5
+        else
+          1
+        end
+      end
+
+      def format_null(column_def)
+        return "NULL" if column_def.nil? || column_def.null
+
+        case column_def.kind
+        when :integer, :float, :boolean
+          "0"
+        else
+          "''"
+        end
+      end
+
+      def format_datetime(value)
+        "'#{Time.iso8601(value.to_s).utc.strftime("%Y-%m-%d %H:%M:%S")}'"
+      rescue ArgumentError
+        SqlText.escape_sql_string(value.to_s)
+      end
+    end
+  end
+end